Geth main/v1.11.2 #323
15
.github/workflows/checks.yml
vendored
Normal file
15
.github/workflows/checks.yml
vendored
Normal file
@ -0,0 +1,15 @@
|
||||
name: checks
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
linter-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: "1.18"
|
||||
check-latest: true
|
||||
- uses: actions/checkout@v2
|
||||
- name: Run linter
|
||||
run: go run build/ci.go lint
|
||||
21
.github/workflows/manual_binary_publish.yaml
vendored
Normal file
21
.github/workflows/manual_binary_publish.yaml
vendored
Normal file
@ -0,0 +1,21 @@
|
||||
name: MANUAL Override Publish geth binary to release
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
giteaPublishTag:
|
||||
description: 'Package to publish TO on gitea; e.g. v1.10.25-statediff-4.2.1-alpha'
|
||||
required: true
|
||||
cercContainerTag:
|
||||
description: 'Tagged Container to extract geth binary FROM'
|
||||
required: true
|
||||
jobs:
|
||||
build:
|
||||
name: Manual override publish of geth binary FROM tagged release TO TAGGED package on git.vdb.to
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Copy ethereum binary file
|
||||
run: docker run --rm --entrypoint cat git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{ github.event.inputs.cercContainerTag }} /usr/local/bin/geth > geth-linux-amd64
|
||||
- name: curl
|
||||
uses: enflo/curl-action@master
|
||||
with:
|
||||
curl: --user cerccicd:${{ secrets.GITEA_TOKEN }} --upload-file geth-linux-amd64 https://git.vdb.to/api/packages/cerc-io/generic/go-ethereum/${{ github.event.inputs.giteaPublishTag }}/geth-linux-amd64
|
||||
21
.github/workflows/manual_publish.yaml
vendored
Normal file
21
.github/workflows/manual_publish.yaml
vendored
Normal file
@ -0,0 +1,21 @@
|
||||
name: MANUAL Override Publish geth binary to release
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
giteaPublishTag:
|
||||
description: 'Package to publish TO on gitea; e.g. v1.10.25-statediff-4.2.1-alpha'
|
||||
required: true
|
||||
cercContainerTag:
|
||||
description: 'Tagged Container to extract geth binary FROM'
|
||||
required: true
|
||||
jobs:
|
||||
build:
|
||||
name: Manual override publish of geth binary FROM tagged release TO TAGGED package on git.vdb.to
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Copy ethereum binary file
|
||||
run: docker run --rm --entrypoint cat git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{ github.event.inputs.cercContainerTag }} /usr/local/bin/geth > geth-linux-amd64
|
||||
- name: curl
|
||||
uses: enflo/curl-action@master
|
||||
with:
|
||||
curl: --user circcicd:${{ secrets.GITEA_TOKEN }} --upload-file geth-linux-amd64 https://git.vdb.to/api/packages/cerc-io/generic/go-ethereum/${{ github.event.inputs.giteaPublishTag }}/geth-linux-amd64
|
||||
145
.github/workflows/old_tests.yml
vendored
Normal file
145
.github/workflows/old_tests.yml
vendored
Normal file
@ -0,0 +1,145 @@
|
||||
name: Tests for Geth that are used in multiple jobs.
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || '37da42481357102a4e02156ad3b178cc5d1f9a42'}}
|
||||
ipld-eth-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || 'be345e0733d2c025e4082c5154e441317ae94cf7' }}
|
||||
GOPATH: /tmp/go
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Run docker build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Run docker build
|
||||
run: docker build -t cerc-io/go-ethereum .
|
||||
|
||||
geth-unit-test:
|
||||
name: Run geth unit test
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GO111MODULE: on
|
||||
steps:
|
||||
- name: Create GOPATH
|
||||
run: mkdir -p /tmp/go
|
||||
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: "1.18"
|
||||
check-latest: true
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
make test
|
||||
|
||||
statediff-unit-test:
|
||||
name: Run state diff unit test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create GOPATH
|
||||
run: mkdir -p /tmp/go
|
||||
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: "1.18"
|
||||
check-latest: true
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Run docker compose
|
||||
run: |
|
||||
docker-compose up -d
|
||||
|
||||
- name: Give the migration a few seconds
|
||||
run: sleep 30;
|
||||
|
||||
- name: Run unit tests
|
||||
run: make statedifftest
|
||||
|
||||
private-network-test:
|
||||
name: Start Geth in a private network.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create GOPATH
|
||||
run: mkdir -p /tmp/go
|
||||
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: "1.18"
|
||||
check-latest: true
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: "./go-ethereum"
|
||||
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ env.stack-orchestrator-ref }}
|
||||
path: "./stack-orchestrator/"
|
||||
repository: cerc-io/mshaw_stack_hack
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ env.ipld-eth-db-ref }}
|
||||
repository: cerc-io/ipld-eth-db
|
||||
path: "./ipld-eth-db/"
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Create config file
|
||||
run: |
|
||||
echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > $GITHUB_WORKSPACE/config.sh
|
||||
echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> $GITHUB_WORKSPACE/config.sh
|
||||
echo db_write=true >> $GITHUB_WORKSPACE/config.sh
|
||||
echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> $GITHUB_WORKSPACE/config.sh
|
||||
cat $GITHUB_WORKSPACE/config.sh
|
||||
|
||||
- name: Compile Geth
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
|
||||
./compile-geth.sh -e docker -p $GITHUB_WORKSPACE/config.sh
|
||||
cd -
|
||||
|
||||
- name: Run docker compose
|
||||
run: |
|
||||
docker-compose \
|
||||
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
|
||||
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \
|
||||
--env-file $GITHUB_WORKSPACE/config.sh \
|
||||
up -d --build
|
||||
|
||||
- name: Make sure the /root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS exists within a certain time frame.
|
||||
shell: bash
|
||||
run: |
|
||||
COUNT=0
|
||||
ATTEMPTS=15
|
||||
docker ps
|
||||
sleep 30;
|
||||
docker logs local_go-ethereum_1
|
||||
docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec go-ethereum ps aux
|
||||
until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" cp go-ethereum:/root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS ./STATEFUL_TEST_DEPLOYED_ADDRESS) || [[ $COUNT -eq $ATTEMPTS ]]; do echo -e "$(( COUNT++ ))... \c"; sleep 10; done
|
||||
[[ $COUNT -eq $ATTEMPTS ]] && echo "Could not find the successful contract deployment" && (exit 1)
|
||||
cat ./STATEFUL_TEST_DEPLOYED_ADDRESS
|
||||
sleep 15;
|
||||
|
||||
- name: Create a new transaction.
|
||||
shell: bash
|
||||
run: |
|
||||
docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec go-ethereum /bin/bash /root/transaction_info/NEW_TRANSACTION
|
||||
echo $?
|
||||
|
||||
- name: Make sure we see entries in the header table
|
||||
shell: bash
|
||||
run: |
|
||||
rows=$(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec ipld-eth-db psql -U vdbm -d vulcanize_testing -AXqtc "SELECT COUNT(*) FROM eth.header_cids")
|
||||
[[ "$rows" -lt "1" ]] && echo "We could not find any rows in postgres table." && (exit 1)
|
||||
echo $rows
|
||||
docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec ipld-eth-db psql -U vdbm -d vulcanize_testing -AXqtc "SELECT * FROM eth.header_cids"
|
||||
7
.github/workflows/on-pr.yml
vendored
Normal file
7
.github/workflows/on-pr.yml
vendored
Normal file
@ -0,0 +1,7 @@
|
||||
name: Build and test
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
run-oldtests:
|
||||
uses: ./.github/workflows/old_tests.yml
|
||||
50
.github/workflows/publish.yaml
vendored
Normal file
50
.github/workflows/publish.yaml
vendored
Normal file
@ -0,0 +1,50 @@
|
||||
name: Publish geth to release
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
jobs:
|
||||
run-tests:
|
||||
uses: ./.github/workflows/tests.yml
|
||||
build:
|
||||
name: Run docker build and publish
|
||||
needs: run-tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Run docker build
|
||||
run: docker build -t vulcanize/go-ethereum -f Dockerfile .
|
||||
- name: Get the version
|
||||
id: vars
|
||||
run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
|
||||
- name: Tag docker image
|
||||
run: docker tag vulcanize/go-ethereum docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
|
||||
- name: Docker Login
|
||||
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
|
||||
- name: Docker Push
|
||||
run: docker push docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
|
||||
push_to_registries:
|
||||
name: Publish assets to Release
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
steps:
|
||||
- name: Get the version
|
||||
id: vars
|
||||
run: |
|
||||
echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
|
||||
echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
|
||||
- name: Tag docker image
|
||||
run: docker tag cerc-io/go-ethereum git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
|
||||
- name: Tag docker image
|
||||
run: docker tag git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.tag}}
|
||||
- name: Docker Login
|
||||
run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
|
||||
- name: Docker Push
|
||||
run: docker push git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
|
||||
- name: Docker Push TAGGED
|
||||
run: docker push git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.tag}}
|
||||
- name: Copy ethereum binary file
|
||||
run: docker run --rm --entrypoint cat git.vdb.to/cerc-io/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} /usr/local/bin/geth > geth-linux-amd64
|
||||
- name: curl
|
||||
uses: enflo/curl-action@master
|
||||
with:
|
||||
curl: --user cerccicd:${{ secrets.GITEA_TOKEN }} --upload-file geth-linux-amd64 https://git.vdb.to/api/packages/cerc-io/generic/go-ethereum/${{steps.vars.outputs.tag}}/geth-linux-amd64
|
||||
143
.github/workflows/tests.yml
vendored
Normal file
143
.github/workflows/tests.yml
vendored
Normal file
@ -0,0 +1,143 @@
|
||||
name: Tests for Geth that are used in multiple jobs.
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || 'f2fd766f5400fcb9eb47b50675d2e3b1f2753702'}}
|
||||
ipld-eth-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || 'be345e0733d2c025e4082c5154e441317ae94cf7' }}
|
||||
GOPATH: /tmp/go
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Run docker build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Run docker build
|
||||
run: docker build -t cerc-io/go-ethereum .
|
||||
|
||||
geth-unit-test:
|
||||
name: Run geth unit test
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GO111MODULE: on
|
||||
steps:
|
||||
- name: Create GOPATH
|
||||
run: mkdir -p /tmp/go
|
||||
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: "1.18"
|
||||
check-latest: true
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
make test
|
||||
|
||||
statediff-unit-test:
|
||||
name: Run state diff unit test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create GOPATH
|
||||
run: mkdir -p /tmp/go
|
||||
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: "1.18"
|
||||
check-latest: true
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Run docker compose
|
||||
run: |
|
||||
docker-compose up -d
|
||||
|
||||
- name: Give the migration a few seconds
|
||||
run: sleep 30;
|
||||
|
||||
- name: Run unit tests
|
||||
run: make statedifftest
|
||||
|
||||
private-network-test:
|
||||
name: Start Geth in a private network.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create GOPATH
|
||||
run: mkdir -p /tmp/go
|
||||
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: "1.18"
|
||||
check-latest: true
|
||||
- name: cerc mkdir
|
||||
run: mkdir /home/runner/cerc
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
repository: cerc-io/ipld-eth-db
|
||||
path: "./ipld-eth-db/"
|
||||
ref: a4961e5e597148c0ff6dd7a34958bdf8c469a47e
|
||||
|
||||
- name: symlink for laconic-so
|
||||
run: ln -sf /home/runner/work/go-ethereum/go-ethereum /home/runner/cerc/go-ethereum
|
||||
|
||||
- name: symlink for laconic-so
|
||||
run: ln -sf /home/runner/work/go-ethereum/go-ethereum/ipld-eth-db /home/runner/cerc/ipld-eth-db
|
||||
|
||||
- name: Install stack orchestrator laconic-so
|
||||
run: |
|
||||
curl -L -o laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
|
||||
chmod +x laconic-so
|
||||
|
||||
- name: Checkout supporting repos in workspace
|
||||
run: |
|
||||
./laconic-so setup-repositories --include cerc/ipld-eth-db
|
||||
|
||||
- name: Build containers from repos in workspace
|
||||
run: |
|
||||
./laconic-so build-containers --include cerc/go-ethereum,cerc/go-ethereum-foundry,cerc/ipld-eth-db
|
||||
|
||||
- name: Start containers
|
||||
run: |
|
||||
docker compose -f docker-compose-cicd.yml up -d
|
||||
|
||||
- name: Make sure the /root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS exists within a certain time frame.
|
||||
shell: bash
|
||||
run: |
|
||||
COUNT=0
|
||||
ATTEMPTS=15
|
||||
sleep 60
|
||||
docker logs go-ethereum-go-ethereum-1
|
||||
docker logs go-ethereum-ipld-eth-db-1
|
||||
docker logs go-ethereum-migrations-1
|
||||
docker compose -f docker-compose-cicd.yml exec go-ethereum ps aux
|
||||
until $(docker compose -f docker-compose-cicd.yml cp go-ethereum:/root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS ./STATEFUL_TEST_DEPLOYED_ADDRESS) || [[ $COUNT -eq $ATTEMPTS ]]; do echo -e "$(( COUNT++ ))... \c"; sleep 10; done
|
||||
[[ $COUNT -eq $ATTEMPTS ]] && echo "Could not find the successful contract deployment" && (exit 1)
|
||||
cat ./STATEFUL_TEST_DEPLOYED_ADDRESS
|
||||
sleep 15;
|
||||
|
||||
- name: Create a new transaction.
|
||||
shell: bash
|
||||
run: |
|
||||
docker compose -f docker-compose-cicd.yml exec go-ethereum /bin/bash /root/transaction_info/NEW_TRANSACTION
|
||||
echo $?
|
||||
|
||||
- name: Make sure we see entries in the header table
|
||||
shell: bash
|
||||
run: |
|
||||
rows=$(docker compose -f docker-compose-cicd.yml - exec ipld-eth-db psql -U vdbm -d vulcanize_testing -AXqtc "SELECT COUNT(*) FROM eth.header_cids")
|
||||
[[ "$rows" -lt "1" ]] && echo "We could not find any rows in postgres table." && (exit 1)
|
||||
echo $rows
|
||||
docker compose -f docker-compose-cicd.yml exec ipld-eth-db psql -U vdbm -d vulcanize_testing -AXqtc "SELECT * FROM eth.header_cids"
|
||||
|
||||
- name: Stop containers
|
||||
run: |
|
||||
docker compose -f docker-compose-cicd.yml down
|
||||
15
.gitignore
vendored
15
.gitignore
vendored
@ -47,3 +47,18 @@ profile.cov
|
||||
/dashboard/assets/package-lock.json
|
||||
|
||||
**/yarn-error.log
|
||||
foundry/deployments/local-private-network/geth-linux-amd64
|
||||
foundry/projects/local-private-network/geth-linux-amd64
|
||||
|
||||
# Helpful repos
|
||||
related-repositories/foundry-test/**
|
||||
related-repositories/hive/**
|
||||
related-repositories/ipld-eth-db/**
|
||||
statediff/indexer/database/sql/statediffing_test_file.sql
|
||||
statediff/statediffing_test_file.sql
|
||||
statediff/known_gaps.sql
|
||||
related-repositories/foundry-test/
|
||||
related-repositories/ipld-eth-db/
|
||||
.idea/
|
||||
trie/concurrent_iterator/fixture/chaindata/ancient/*.meta
|
||||
|
||||
|
||||
2
.gitmodules
vendored
2
.gitmodules
vendored
@ -5,4 +5,4 @@
|
||||
[submodule "evm-benchmarks"]
|
||||
path = tests/evm-benchmarks
|
||||
url = https://github.com/ipsilon/evm-benchmarks
|
||||
shallow = true
|
||||
shallow = true
|
||||
7
Dockerfile.amd64
Normal file
7
Dockerfile.amd64
Normal file
@ -0,0 +1,7 @@
|
||||
# Build Geth in a stock Go builder container
|
||||
FROM golang:1.15.5 as builder
|
||||
|
||||
#RUN apk add --no-cache make gcc musl-dev linux-headers git
|
||||
|
||||
ADD . /go-ethereum
|
||||
RUN cd /go-ethereum && make geth
|
||||
50
Jenkinsfile
vendored
Normal file
50
Jenkinsfile
vendored
Normal file
@ -0,0 +1,50 @@
|
||||
pipeline {
|
||||
agent any
|
||||
|
||||
stages {
|
||||
stage('Build') {
|
||||
steps {
|
||||
script{
|
||||
docker.withRegistry('https://git.vdb.to'){
|
||||
echo 'Building geth image...'
|
||||
//def geth_image = docker.build("cerc-io/go-ethereum:jenkinscicd")
|
||||
echo 'built geth image'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Test') {
|
||||
agent {
|
||||
docker {
|
||||
image 'cerc-io/foundation:jenkinscicd'
|
||||
//image 'cerc-io/foundation_alpine:jenkinscicd'
|
||||
}
|
||||
}
|
||||
|
||||
environment {
|
||||
GO111MODULE = "on"
|
||||
CGO_ENABLED = 1
|
||||
//GOPATH = "${JENKINS_HOME}/jobs/${JOB_NAME}/builds/${BUILD_ID}"
|
||||
//GOPATH = "/go"
|
||||
GOPATH = "/tmp/go"
|
||||
//GOMODCACHE = "/go/pkg/mod"
|
||||
GOCACHE = "${WORKSPACE}/.cache/go-build"
|
||||
GOENV = "${WORKSPACE}/.config/go/env"
|
||||
GOMODCACHE = "/tmp/go/pkg/mod"
|
||||
GOWORK=""
|
||||
//GOFLAGS=""
|
||||
|
||||
}
|
||||
steps {
|
||||
echo 'Testing ...'
|
||||
//sh '/usr/local/go/bin/go test -p 1 -v ./...'
|
||||
sh 'make test'
|
||||
}
|
||||
}
|
||||
stage('Packaging') {
|
||||
steps {
|
||||
echo 'Packaging ...'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
33
Makefile
33
Makefile
@ -4,10 +4,31 @@
|
||||
|
||||
.PHONY: geth android ios evm all test clean
|
||||
|
||||
BIN = $(GOPATH)/bin
|
||||
|
||||
## Migration tool
|
||||
GOOSE = $(BIN)/goose
|
||||
$(BIN)/goose:
|
||||
go get -u github.com/pressly/goose/cmd/goose
|
||||
|
||||
GOBIN = ./build/bin
|
||||
GO ?= latest
|
||||
GORUN = env GO111MODULE=on go run
|
||||
|
||||
#Database
|
||||
HOST_NAME = localhost
|
||||
PORT = 5432
|
||||
USER = vdbm
|
||||
PASSWORD = password
|
||||
|
||||
# Set env variable
|
||||
# `PGPASSWORD` is used by `createdb` and `dropdb`
|
||||
export PGPASSWORD=$(PASSWORD)
|
||||
|
||||
#Test
|
||||
TEST_DB = vulcanize_public
|
||||
TEST_CONNECT_STRING = postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
|
||||
|
||||
geth:
|
||||
$(GORUN) build/ci.go install ./cmd/geth
|
||||
@echo "Done building."
|
||||
@ -17,7 +38,7 @@ all:
|
||||
$(GORUN) build/ci.go install
|
||||
|
||||
test: all
|
||||
$(GORUN) build/ci.go test
|
||||
$(GORUN) build/ci.go test -v
|
||||
|
||||
lint: ## Run linters.
|
||||
$(GORUN) build/ci.go lint
|
||||
@ -36,3 +57,13 @@ devtools:
|
||||
env GOBIN= go install ./cmd/abigen
|
||||
@type "solc" 2> /dev/null || echo 'Please install solc'
|
||||
@type "protoc" 2> /dev/null || echo 'Please install protoc'
|
||||
|
||||
.PHONY: statedifftest
|
||||
statedifftest: | $(GOOSE)
|
||||
GO111MODULE=on go get github.com/stretchr/testify/assert@v1.7.0
|
||||
GO111MODULE=on MODE=statediff go test -p 1 ./statediff/... -v
|
||||
|
||||
.PHONY: statediff_filewriting_test
|
||||
statediff_filetest: | $(GOOSE)
|
||||
GO111MODULE=on go get github.com/stretchr/testify/assert@v1.7.0
|
||||
GO111MODULE=on MODE=statediff STATEDIFF_DB=file go test -p 1 ./statediff/... -v
|
||||
|
||||
@ -18,10 +18,12 @@ package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
@ -40,6 +42,12 @@ import (
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethereum/go-ethereum/node"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/statediff"
|
||||
dumpdb "github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
"github.com/naoina/toml"
|
||||
)
|
||||
|
||||
@ -151,6 +159,9 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
|
||||
cfg.Ethstats.URL = ctx.String(utils.EthStatsURLFlag.Name)
|
||||
}
|
||||
applyMetricConfig(ctx, &cfg)
|
||||
if ctx.Bool(utils.StateDiffFlag.Name) {
|
||||
cfg.Eth.Diffing = true
|
||||
}
|
||||
|
||||
return stack, cfg
|
||||
}
|
||||
@ -167,7 +178,110 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
|
||||
// Configure log filter RPC API.
|
||||
filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth)
|
||||
|
||||
// Configure GraphQL if requested.
|
||||
if ctx.Bool(utils.StateDiffFlag.Name) {
|
||||
var indexerConfig interfaces.Config
|
||||
var clientName, nodeID string
|
||||
if ctx.IsSet(utils.StateDiffWritingFlag.Name) {
|
||||
clientName = ctx.String(utils.StateDiffDBClientNameFlag.Name)
|
||||
if ctx.IsSet(utils.StateDiffDBNodeIDFlag.Name) {
|
||||
nodeID = ctx.String(utils.StateDiffDBNodeIDFlag.Name)
|
||||
} else {
|
||||
utils.Fatalf("Must specify node ID for statediff DB output")
|
||||
}
|
||||
|
||||
dbTypeStr := ctx.String(utils.StateDiffDBTypeFlag.Name)
|
||||
dbType, err := shared.ResolveDBType(dbTypeStr)
|
||||
if err != nil {
|
||||
utils.Fatalf("%v", err)
|
||||
}
|
||||
switch dbType {
|
||||
case shared.FILE:
|
||||
fileModeStr := ctx.String(utils.StateDiffFileMode.Name)
|
||||
fileMode, err := file.ResolveFileMode(fileModeStr)
|
||||
if err != nil {
|
||||
utils.Fatalf("%v", err)
|
||||
}
|
||||
|
||||
indexerConfig = file.Config{
|
||||
Mode: fileMode,
|
||||
OutputDir: ctx.String(utils.StateDiffFileCsvDir.Name),
|
||||
FilePath: ctx.String(utils.StateDiffFilePath.Name),
|
||||
WatchedAddressesFilePath: ctx.String(utils.StateDiffWatchedAddressesFilePath.Name),
|
||||
}
|
||||
case shared.POSTGRES:
|
||||
driverTypeStr := ctx.String(utils.StateDiffDBDriverTypeFlag.Name)
|
||||
driverType, err := postgres.ResolveDriverType(driverTypeStr)
|
||||
if err != nil {
|
||||
utils.Fatalf("%v", err)
|
||||
}
|
||||
pgConfig := postgres.Config{
|
||||
Hostname: ctx.String(utils.StateDiffDBHostFlag.Name),
|
||||
Port: ctx.Int(utils.StateDiffDBPortFlag.Name),
|
||||
DatabaseName: ctx.String(utils.StateDiffDBNameFlag.Name),
|
||||
Username: ctx.String(utils.StateDiffDBUserFlag.Name),
|
||||
Password: ctx.String(utils.StateDiffDBPasswordFlag.Name),
|
||||
ID: nodeID,
|
||||
ClientName: clientName,
|
||||
Driver: driverType,
|
||||
}
|
||||
if ctx.IsSet(utils.StateDiffUpsert.Name) {
|
||||
pgConfig.Upsert = ctx.Bool(utils.StateDiffUpsert.Name)
|
||||
}
|
||||
if ctx.IsSet(utils.StateDiffDBMinConns.Name) {
|
||||
pgConfig.MinConns = ctx.Int(utils.StateDiffDBMinConns.Name)
|
||||
}
|
||||
if ctx.IsSet(utils.StateDiffDBMaxConns.Name) {
|
||||
pgConfig.MaxConns = ctx.Int(utils.StateDiffDBMaxConns.Name)
|
||||
}
|
||||
if ctx.IsSet(utils.StateDiffDBMaxIdleConns.Name) {
|
||||
pgConfig.MaxIdle = ctx.Int(utils.StateDiffDBMaxIdleConns.Name)
|
||||
}
|
||||
if ctx.IsSet(utils.StateDiffDBMaxConnLifetime.Name) {
|
||||
pgConfig.MaxConnLifetime = time.Duration(ctx.Duration(utils.StateDiffDBMaxConnLifetime.Name).Seconds())
|
||||
}
|
||||
if ctx.IsSet(utils.StateDiffDBMaxConnIdleTime.Name) {
|
||||
pgConfig.MaxConnIdleTime = time.Duration(ctx.Duration(utils.StateDiffDBMaxConnIdleTime.Name).Seconds())
|
||||
}
|
||||
if ctx.IsSet(utils.StateDiffDBConnTimeout.Name) {
|
||||
pgConfig.ConnTimeout = time.Duration(ctx.Duration(utils.StateDiffDBConnTimeout.Name).Seconds())
|
||||
}
|
||||
if ctx.IsSet(utils.StateDiffLogStatements.Name) {
|
||||
pgConfig.LogStatements = ctx.Bool(utils.StateDiffLogStatements.Name)
|
||||
}
|
||||
indexerConfig = pgConfig
|
||||
case shared.DUMP:
|
||||
dumpTypeStr := ctx.String(utils.StateDiffDBDumpDst.Name)
|
||||
dumpType, err := dumpdb.ResolveDumpType(dumpTypeStr)
|
||||
if err != nil {
|
||||
utils.Fatalf("%v", err)
|
||||
}
|
||||
switch dumpType {
|
||||
case dumpdb.STDERR:
|
||||
indexerConfig = dumpdb.Config{Dump: os.Stdout}
|
||||
case dumpdb.STDOUT:
|
||||
indexerConfig = dumpdb.Config{Dump: os.Stderr}
|
||||
case dumpdb.DISCARD:
|
||||
indexerConfig = dumpdb.Config{Dump: dumpdb.NewDiscardWriterCloser()}
|
||||
default:
|
||||
utils.Fatalf("unrecognized dump destination: %s", dumpType)
|
||||
}
|
||||
default:
|
||||
utils.Fatalf("unrecognized database type: %s", dbType)
|
||||
}
|
||||
}
|
||||
p := statediff.Config{
|
||||
IndexerConfig: indexerConfig,
|
||||
ID: nodeID,
|
||||
ClientName: clientName,
|
||||
Context: context.Background(),
|
||||
EnableWriteLoop: ctx.Bool(utils.StateDiffWritingFlag.Name),
|
||||
NumWorkers: ctx.Uint(utils.StateDiffWorkersFlag.Name),
|
||||
WaitForSync: ctx.Bool(utils.StateDiffWaitForSync.Name),
|
||||
}
|
||||
utils.RegisterStateDiffService(stack, eth, &cfg.Eth, p, backend)
|
||||
}
|
||||
|
||||
// Configure GraphQL if requested
|
||||
if ctx.IsSet(utils.GraphQLEnabledFlag.Name) {
|
||||
utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node)
|
||||
}
|
||||
|
||||
@ -149,6 +149,32 @@ var (
|
||||
utils.GpoMaxGasPriceFlag,
|
||||
utils.GpoIgnoreGasPriceFlag,
|
||||
utils.MinerNotifyFullFlag,
|
||||
utils.StateDiffFlag,
|
||||
utils.StateDiffDBTypeFlag,
|
||||
utils.StateDiffDBDriverTypeFlag,
|
||||
utils.StateDiffDBDumpDst,
|
||||
utils.StateDiffDBNameFlag,
|
||||
utils.StateDiffDBPasswordFlag,
|
||||
utils.StateDiffDBUserFlag,
|
||||
utils.StateDiffDBHostFlag,
|
||||
utils.StateDiffDBPortFlag,
|
||||
utils.StateDiffDBMaxConnLifetime,
|
||||
utils.StateDiffDBMaxConnIdleTime,
|
||||
utils.StateDiffDBMaxConns,
|
||||
utils.StateDiffDBMinConns,
|
||||
utils.StateDiffDBMaxIdleConns,
|
||||
utils.StateDiffDBConnTimeout,
|
||||
utils.StateDiffDBNodeIDFlag,
|
||||
utils.StateDiffDBClientNameFlag,
|
||||
utils.StateDiffWritingFlag,
|
||||
utils.StateDiffWorkersFlag,
|
||||
utils.StateDiffFileMode,
|
||||
utils.StateDiffFileCsvDir,
|
||||
utils.StateDiffFilePath,
|
||||
utils.StateDiffWaitForSync,
|
||||
utils.StateDiffWatchedAddressesFilePath,
|
||||
utils.StateDiffUpsert,
|
||||
utils.StateDiffLogStatements,
|
||||
configFileFlag,
|
||||
}, utils.NetworkFlags, utils.DatabasePathFlags)
|
||||
|
||||
|
||||
@ -74,6 +74,8 @@ import (
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
"github.com/ethereum/go-ethereum/statediff"
|
||||
|
||||
pcsclite "github.com/gballet/go-libpcsclite"
|
||||
gopsutil "github.com/shirou/gopsutil/mem"
|
||||
"github.com/urfave/cli/v2"
|
||||
@ -994,6 +996,123 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
|
||||
Value: metrics.DefaultConfig.InfluxDBOrganization,
|
||||
Category: flags.MetricsCategory,
|
||||
}
|
||||
|
||||
StateDiffFlag = &cli.BoolFlag{
|
||||
Name: "statediff",
|
||||
Usage: "Enables the processing of state diffs between each block",
|
||||
Category: flags.MiscCategory,
|
||||
}
|
||||
StateDiffDBTypeFlag = &cli.StringFlag{
|
||||
Name: "statediff.db.type",
|
||||
Usage: "Statediff database type (current options: postgres, file, dump)",
|
||||
Value: "postgres",
|
||||
}
|
||||
StateDiffDBDriverTypeFlag = &cli.StringFlag{
|
||||
Name: "statediff.db.driver",
|
||||
Usage: "Statediff database driver type",
|
||||
Value: "pgx",
|
||||
}
|
||||
StateDiffDBDumpDst = &cli.StringFlag{
|
||||
Name: "statediff.dump.dst",
|
||||
Usage: "Statediff database dump destination (default is stdout)",
|
||||
Value: "stdout",
|
||||
}
|
||||
StateDiffDBHostFlag = &cli.StringFlag{
|
||||
Name: "statediff.db.host",
|
||||
Usage: "Statediff database hostname/ip",
|
||||
Value: "localhost",
|
||||
}
|
||||
StateDiffDBPortFlag = &cli.IntFlag{
|
||||
Name: "statediff.db.port",
|
||||
Usage: "Statediff database port",
|
||||
Value: 5432,
|
||||
}
|
||||
StateDiffDBNameFlag = &cli.StringFlag{
|
||||
Name: "statediff.db.name",
|
||||
Usage: "Statediff database name",
|
||||
}
|
||||
StateDiffDBPasswordFlag = &cli.StringFlag{
|
||||
Name: "statediff.db.password",
|
||||
Usage: "Statediff database password",
|
||||
}
|
||||
StateDiffDBUserFlag = &cli.StringFlag{
|
||||
Name: "statediff.db.user",
|
||||
Usage: "Statediff database username",
|
||||
Value: "postgres",
|
||||
}
|
||||
StateDiffDBMaxConnLifetime = &cli.DurationFlag{
|
||||
Name: "statediff.db.maxconnlifetime",
|
||||
Usage: "Statediff database maximum connection lifetime (in seconds)",
|
||||
}
|
||||
StateDiffDBMaxConnIdleTime = &cli.DurationFlag{
|
||||
Name: "statediff.db.maxconnidletime",
|
||||
Usage: "Statediff database maximum connection idle time (in seconds)",
|
||||
}
|
||||
StateDiffDBMaxConns = &cli.IntFlag{
|
||||
Name: "statediff.db.maxconns",
|
||||
Usage: "Statediff database maximum connections",
|
||||
}
|
||||
StateDiffDBMinConns = &cli.IntFlag{
|
||||
Name: "statediff.db.minconns",
|
||||
Usage: "Statediff database minimum connections",
|
||||
}
|
||||
StateDiffDBMaxIdleConns = &cli.IntFlag{
|
||||
Name: "statediff.db.maxidleconns",
|
||||
Usage: "Statediff database maximum idle connections",
|
||||
}
|
||||
StateDiffDBConnTimeout = &cli.DurationFlag{
|
||||
Name: "statediff.db.conntimeout",
|
||||
Usage: "Statediff database connection timeout (in seconds)",
|
||||
}
|
||||
StateDiffDBNodeIDFlag = &cli.StringFlag{
|
||||
Name: "statediff.db.nodeid",
|
||||
Usage: "Node ID to use when writing state diffs to database",
|
||||
}
|
||||
StateDiffFileMode = &cli.StringFlag{
|
||||
Name: "statediff.file.mode",
|
||||
Usage: "Statediff file writing mode (current options: csv, sql)",
|
||||
Value: "csv",
|
||||
}
|
||||
StateDiffFileCsvDir = &cli.StringFlag{
|
||||
Name: "statediff.file.csvdir",
|
||||
Usage: "Full path of output directory to write statediff data out to when operating in csv file mode",
|
||||
}
|
||||
StateDiffFilePath = &cli.StringFlag{
|
||||
Name: "statediff.file.path",
|
||||
Usage: "Full path (including filename) to write statediff data out to when operating in sql file mode",
|
||||
}
|
||||
StateDiffWatchedAddressesFilePath = &cli.StringFlag{
|
||||
Name: "statediff.file.wapath",
|
||||
Usage: "Full path (including filename) to write statediff watched addresses out to when operating in file mode",
|
||||
}
|
||||
StateDiffDBClientNameFlag = &cli.StringFlag{
|
||||
Name: "statediff.db.clientname",
|
||||
Usage: "Client name to use when writing state diffs to database",
|
||||
Value: "go-ethereum",
|
||||
}
|
||||
StateDiffUpsert = &cli.BoolFlag{
|
||||
Name: "statediff.db.upsert",
|
||||
Usage: "Should the statediff service overwrite data existing in the database?",
|
||||
Value: false,
|
||||
}
|
||||
StateDiffLogStatements = &cli.BoolFlag{
|
||||
Name: "statediff.db.logstatements",
|
||||
Usage: "Should the statediff service log all database statements? (Note: pgx only)",
|
||||
Value: false,
|
||||
}
|
||||
StateDiffWritingFlag = &cli.BoolFlag{
|
||||
Name: "statediff.writing",
|
||||
Usage: "Activates progressive writing of state diffs to database as new block are synced",
|
||||
}
|
||||
StateDiffWorkersFlag = &cli.UintFlag{
|
||||
Name: "statediff.workers",
|
||||
Usage: "Number of concurrent workers to use during statediff processing (default 1)",
|
||||
Value: 1,
|
||||
}
|
||||
StateDiffWaitForSync = &cli.BoolFlag{
|
||||
Name: "statediff.waitforsync",
|
||||
Usage: "Should the statediff service wait for geth to catch up to the head of the chain?",
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
@ -1246,6 +1365,10 @@ func setWS(ctx *cli.Context, cfg *node.Config) {
|
||||
if ctx.IsSet(WSPathPrefixFlag.Name) {
|
||||
cfg.WSPathPrefix = ctx.String(WSPathPrefixFlag.Name)
|
||||
}
|
||||
|
||||
if ctx.Bool(StateDiffFlag.Name) {
|
||||
cfg.WSModules = append(cfg.WSModules, "statediff")
|
||||
}
|
||||
}
|
||||
|
||||
// setIPC creates an IPC path configuration from the set command line flags,
|
||||
@ -2017,6 +2140,15 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend
|
||||
return backend.APIBackend, backend
|
||||
}
|
||||
|
||||
// RegisterLesEthService adds an Ethereum les client to the stack.
|
||||
func RegisterLesEthService(stack *node.Node, cfg *eth.Config) *les.LightEthereum {
|
||||
backend, err := les.New(stack, cfg)
|
||||
if err != nil {
|
||||
Fatalf("Failed to register the Ethereum service: %v", err)
|
||||
}
|
||||
return backend
|
||||
}
|
||||
|
||||
// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to the node.
|
||||
func RegisterEthStatsService(stack *node.Node, backend ethapi.Backend, url string) {
|
||||
if err := ethstats.New(stack, backend, backend.Engine(), url); err != nil {
|
||||
@ -2063,6 +2195,13 @@ func RegisterFullSyncTester(stack *node.Node, eth *eth.Ethereum, path string) {
|
||||
log.Info("Registered full-sync tester", "number", block.NumberU64(), "hash", block.Hash())
|
||||
}
|
||||
|
||||
// RegisterStateDiffService configures and registers a service to stream state diff data over RPC
|
||||
func RegisterStateDiffService(stack *node.Node, ethServ *eth.Ethereum, cfg *ethconfig.Config, params statediff.Config, backend ethapi.Backend) {
|
||||
if err := statediff.New(stack, ethServ, cfg, params, backend); err != nil {
|
||||
Fatalf("Failed to register the Statediff service: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func SetupMetrics(ctx *cli.Context) {
|
||||
if metrics.Enabled {
|
||||
log.Info("Enabling metrics collection")
|
||||
|
||||
@ -31,6 +31,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/consensus/misc"
|
||||
"github.com/ethereum/go-ethereum/core/state"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
@ -333,7 +334,13 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, pa
|
||||
// the difficulty that a new block should have when created at time
|
||||
// given the parent block's time and difficulty.
|
||||
func (ethash *Ethash) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64, parent *types.Header) *big.Int {
|
||||
return CalcDifficulty(chain.Config(), time, parent)
|
||||
var config = chain.Config()
|
||||
var ret = CalcDifficulty(config, time, parent)
|
||||
if nil != config.CappedMaximumDifficulty && ret.Cmp(config.CappedMaximumDifficulty) >= 0 {
|
||||
log.Info(fmt.Sprintf("Using capped difficulty %d", config.CappedMaximumDifficulty))
|
||||
return config.CappedMaximumDifficulty
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// CalcDifficulty is the difficulty adjustment algorithm. It returns
|
||||
|
||||
@ -139,6 +139,7 @@ type CacheConfig struct {
|
||||
|
||||
SnapshotNoBuild bool // Whether the background generation is allowed
|
||||
SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
|
||||
StateDiffing bool // Whether or not the statediffing service is running
|
||||
}
|
||||
|
||||
// defaultCacheConfig are the default caching values if none are specified by the
|
||||
@ -224,6 +225,10 @@ type BlockChain struct {
|
||||
processor Processor // Block transaction processor interface
|
||||
forker *ForkChoice
|
||||
vmConfig vm.Config
|
||||
|
||||
// Locked roots and their mutex
|
||||
trieLock sync.Mutex
|
||||
lockedRoots map[common.Hash]bool
|
||||
}
|
||||
|
||||
// NewBlockChain returns a fully initialised block chain using information
|
||||
@ -272,6 +277,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
|
||||
futureBlocks: lru.NewCache[common.Hash, *types.Block](maxFutureBlocks),
|
||||
engine: engine,
|
||||
vmConfig: vmConfig,
|
||||
lockedRoots: make(map[common.Hash]bool),
|
||||
}
|
||||
bc.forker = NewForkChoice(bc, shouldPreserve)
|
||||
bc.stateCache = state.NewDatabaseWithNodeDB(bc.db, bc.triedb)
|
||||
@ -957,7 +963,10 @@ func (bc *BlockChain) Stop() {
|
||||
}
|
||||
}
|
||||
for !bc.triegc.Empty() {
|
||||
triedb.Dereference(bc.triegc.PopItem())
|
||||
pruneRoot := bc.triegc.PopItem()
|
||||
if !bc.TrieLocked(pruneRoot) {
|
||||
triedb.Dereference(pruneRoot)
|
||||
}
|
||||
}
|
||||
if size, _ := triedb.Size(); size != 0 {
|
||||
log.Error("Dangling trie nodes after full cleanup")
|
||||
@ -1349,6 +1358,10 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
|
||||
bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
|
||||
bc.triegc.Push(root, -int64(block.NumberU64()))
|
||||
|
||||
if bc.cacheConfig.StateDiffing {
|
||||
bc.LockTrie(root)
|
||||
}
|
||||
|
||||
current := block.NumberU64()
|
||||
// Flush limits are not considered for the first TriesInMemory blocks.
|
||||
if current <= TriesInMemory {
|
||||
@ -2474,3 +2487,28 @@ func (bc *BlockChain) SetBlockValidatorAndProcessorForTesting(v Validator, p Pro
|
||||
func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) {
|
||||
atomic.StoreInt64(&bc.flushInterval, int64(interval))
|
||||
}
|
||||
|
||||
// TrieLocked returns whether the trie associated with the provided root is locked for use
|
||||
func (bc *BlockChain) TrieLocked(root common.Hash) bool {
|
||||
bc.trieLock.Lock()
|
||||
locked, ok := bc.lockedRoots[root]
|
||||
bc.trieLock.Unlock()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return locked
|
||||
}
|
||||
|
||||
// LockTrie prevents dereferencing of the provided root
|
||||
func (bc *BlockChain) LockTrie(root common.Hash) {
|
||||
bc.trieLock.Lock()
|
||||
bc.lockedRoots[root] = true
|
||||
bc.trieLock.Unlock()
|
||||
}
|
||||
|
||||
// UnlockTrie allows dereferencing of the provided root- provided it was previously locked
|
||||
func (bc *BlockChain) UnlockTrie(root common.Hash) {
|
||||
bc.trieLock.Lock()
|
||||
bc.lockedRoots[root] = false
|
||||
bc.trieLock.Unlock()
|
||||
}
|
||||
|
||||
@ -643,6 +643,23 @@ func ReadReceipts(db ethdb.Reader, hash common.Hash, number uint64, config *para
|
||||
return receipts
|
||||
}
|
||||
|
||||
// ReadStorageReceipts is a modification of ReadRawReceipts that skips a conversion to types.Receipt
|
||||
// and returns the ReceipstForStorage instead
|
||||
func ReadStorageReceipts(db ethdb.Reader, hash common.Hash, number uint64) []*types.ReceiptForStorage {
|
||||
// Retrieve the flattened receipt slice
|
||||
data := ReadReceiptsRLP(db, hash, number)
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Convert the receipts from their storage form to their internal representation
|
||||
var storageReceipts []*types.ReceiptForStorage
|
||||
if err := rlp.DecodeBytes(data, &storageReceipts); err != nil {
|
||||
log.Error("Invalid receipt array RLP", "hash", hash, "err", err)
|
||||
return nil
|
||||
}
|
||||
return storageReceipts
|
||||
}
|
||||
|
||||
// WriteReceipts stores all the transaction receipts belonging to a block.
|
||||
func WriteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64, receipts types.Receipts) {
|
||||
// Convert the receipts into their storage form and serialize them
|
||||
@ -785,6 +802,17 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts
|
||||
})
|
||||
}
|
||||
|
||||
// WriteAncientBlock is a public wrapper of writeAncientBlock
|
||||
func WriteAncientBlock(op ethdb.AncientWriteOp, block *types.Block, receipts types.Receipts, td *big.Int) error {
|
||||
stReceipts := make([]*types.ReceiptForStorage, len(receipts))
|
||||
// Convert receipts to storage format and sum up total difficulty.
|
||||
stReceipts = stReceipts[:0]
|
||||
for i, receipt := range receipts {
|
||||
stReceipts[i] = (*types.ReceiptForStorage)(receipt)
|
||||
}
|
||||
return writeAncientBlock(op, block, block.Header(), stReceipts, td)
|
||||
}
|
||||
|
||||
func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error {
|
||||
num := block.NumberU64()
|
||||
if err := op.AppendRaw(ChainFreezerHashTable, num, block.Hash().Bytes()); err != nil {
|
||||
|
||||
@ -69,6 +69,7 @@ type Receipt struct {
|
||||
BlockHash common.Hash `json:"blockHash,omitempty"`
|
||||
BlockNumber *big.Int `json:"blockNumber,omitempty"`
|
||||
TransactionIndex uint `json:"transactionIndex"`
|
||||
LogRoot common.Hash `json:"logRoot"`
|
||||
}
|
||||
|
||||
type receiptMarshaling struct {
|
||||
@ -114,6 +115,9 @@ func NewReceipt(root []byte, failed bool, cumulativeGasUsed uint64) *Receipt {
|
||||
|
||||
// EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt
|
||||
// into an RLP stream. If no post state is present, byzantium fork is assumed.
|
||||
// For a legacy Receipt this returns RLP([PostStateOrStatus, CumulativeGasUsed, Bloom, Logs])
|
||||
// For a EIP-2718 Receipt this returns RLP(TxType || ReceiptPayload)
|
||||
// For a EIP-2930 Receipt, TxType == 0x01 and ReceiptPayload == RLP([PostStateOrStatus, CumulativeGasUsed, Bloom, Logs])
|
||||
func (r *Receipt) EncodeRLP(w io.Writer) error {
|
||||
data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs}
|
||||
if r.Type == LegacyTxType {
|
||||
|
||||
@ -88,6 +88,9 @@ type TxData interface {
|
||||
}
|
||||
|
||||
// EncodeRLP implements rlp.Encoder
|
||||
// For a legacy Transaction this returns RLP([AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, V, R, S])
|
||||
// For a EIP-2718 Transaction this returns RLP(TxType || TxPayload)
|
||||
// For a EIP-2930 Transaction, TxType == 0x01 and TxPayload == RLP([ChainID, AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, AccessList, V, R, S]
|
||||
func (tx *Transaction) EncodeRLP(w io.Writer) error {
|
||||
if tx.Type() == LegacyTxType {
|
||||
return rlp.Encode(w, tx.inner)
|
||||
@ -108,9 +111,10 @@ func (tx *Transaction) encodeTyped(w *bytes.Buffer) error {
|
||||
return rlp.Encode(w, tx.inner)
|
||||
}
|
||||
|
||||
// MarshalBinary returns the canonical encoding of the transaction.
|
||||
// For legacy transactions, it returns the RLP encoding. For EIP-2718 typed
|
||||
// transactions, it returns the type and payload.
|
||||
// MarshalBinary returns the canonical consensus encoding of the transaction.
|
||||
// For a legacy Transaction this returns RLP([AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, V, R, S])
|
||||
// For a EIP-2718 Transaction this returns TxType || TxPayload
|
||||
// For a EIP-2930 Transaction, TxType == 0x01 and TxPayload == RLP([ChainID, AccountNonce, GasPrice, GasLimit, Recipient, Amount, Data, AccessList, V, R, S]
|
||||
func (tx *Transaction) MarshalBinary() ([]byte, error) {
|
||||
if tx.Type() == LegacyTxType {
|
||||
return rlp.EncodeToBytes(tx.inner)
|
||||
|
||||
54
docker-compose-cicd.yml
Normal file
54
docker-compose-cicd.yml
Normal file
@ -0,0 +1,54 @@
|
||||
version: "3.2"
|
||||
|
||||
services:
|
||||
go-ethereum:
|
||||
restart: always
|
||||
depends_on:
|
||||
- migrations
|
||||
image: cerc/go-ethereum-foundry:local
|
||||
healthcheck:
|
||||
test: [ "CMD", "nc", "-v", "localhost", "8545" ]
|
||||
interval: 30s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
environment:
|
||||
DB_USER: vdbm
|
||||
DB_NAME: vulcanize_testing
|
||||
DB_HOST: ipld-eth-db
|
||||
DB_PORT: 5432
|
||||
DB_PASSWORD: password
|
||||
DB_WRITE: "true"
|
||||
DB_TYPE: postgres
|
||||
DB_DRIVER: sqlx
|
||||
DB_WAIT_FOR_SYNC: "true"
|
||||
CHAIN_ID: "99"
|
||||
USE_GENESIS: "true"
|
||||
EXTRA_START_ARGS: ${extra_args}
|
||||
ports:
|
||||
- "127.0.0.1:8545:8545"
|
||||
- "127.0.0.1:8546:8546"
|
||||
|
||||
migrations:
|
||||
restart: on-failure
|
||||
depends_on:
|
||||
- ipld-eth-db
|
||||
image: cerc/ipld-eth-db:local
|
||||
environment:
|
||||
DATABASE_USER: "vdbm"
|
||||
DATABASE_NAME: "vulcanize_testing"
|
||||
DATABASE_PASSWORD: "password"
|
||||
DATABASE_HOSTNAME: "ipld-eth-db"
|
||||
DATABASE_PORT: 5432
|
||||
|
||||
ipld-eth-db:
|
||||
image: timescale/timescaledb:latest-pg14
|
||||
restart: on-failure
|
||||
command: ["postgres", "-c", "log_statement=all"]
|
||||
environment:
|
||||
POSTGRES_USER: "vdbm"
|
||||
POSTGRES_DB: "vulcanize_testing"
|
||||
POSTGRES_PASSWORD: "password"
|
||||
ports:
|
||||
- "127.0.0.1:8077:5432"
|
||||
volumes:
|
||||
- ./statediff/indexer/database/file:/file_indexer
|
||||
27
docker-compose.yml
Normal file
27
docker-compose.yml
Normal file
@ -0,0 +1,27 @@
|
||||
version: "3.2"
|
||||
|
||||
services:
|
||||
migrations:
|
||||
restart: on-failure
|
||||
depends_on:
|
||||
- ipld-eth-db
|
||||
image: vulcanize/ipld-eth-db:v4.2.1-alpha
|
||||
environment:
|
||||
DATABASE_USER: "vdbm"
|
||||
DATABASE_NAME: "vulcanize_testing"
|
||||
DATABASE_PASSWORD: "password"
|
||||
DATABASE_HOSTNAME: "ipld-eth-db"
|
||||
DATABASE_PORT: 5432
|
||||
|
||||
ipld-eth-db:
|
||||
image: timescale/timescaledb:latest-pg14
|
||||
restart: always
|
||||
command: ["postgres", "-c", "log_statement=all"]
|
||||
environment:
|
||||
POSTGRES_USER: "vdbm"
|
||||
POSTGRES_DB: "vulcanize_testing"
|
||||
POSTGRES_PASSWORD: "password"
|
||||
ports:
|
||||
- "127.0.0.1:8077:5432"
|
||||
volumes:
|
||||
- ./statediff/indexer/database/file:/file_indexer
|
||||
@ -191,6 +191,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
|
||||
TrieTimeLimit: config.TrieTimeout,
|
||||
SnapshotLimit: config.SnapshotCache,
|
||||
Preimages: config.Preimages,
|
||||
StateDiffing: config.Diffing,
|
||||
}
|
||||
)
|
||||
// Override the chain config with provided settings.
|
||||
|
||||
@ -207,6 +207,10 @@ type Config struct {
|
||||
|
||||
// OverrideShanghai (TODO: remove after the fork)
|
||||
OverrideShanghai *uint64 `toml:",omitempty"`
|
||||
|
||||
// Signify whether or not we are producing statediffs
|
||||
// If we are, do not dereference state roots until the statediffing service is done with them
|
||||
Diffing bool
|
||||
}
|
||||
|
||||
// CreateConsensusEngine creates a consensus engine for the given chain configuration.
|
||||
|
||||
@ -81,6 +81,7 @@ const (
|
||||
// minTrienodeHealThrottle is the minimum divisor for throttling trie node
|
||||
// heal requests to avoid overloading the local node and excessively expanding
|
||||
// the state trie breadth wise.
|
||||
|
||||
minTrienodeHealThrottle = 1
|
||||
|
||||
// maxTrienodeHealThrottle is the maximum divisor for throttling trie node
|
||||
|
||||
56
go.mod
56
go.mod
@ -40,6 +40,14 @@ require (
|
||||
github.com/huin/goupnp v1.0.3
|
||||
github.com/influxdata/influxdb v1.8.3
|
||||
github.com/influxdata/influxdb-client-go/v2 v2.4.0
|
||||
github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
|
||||
github.com/ipfs/go-block-format v0.0.3
|
||||
github.com/ipfs/go-cid v0.2.0
|
||||
github.com/ipfs/go-ipfs-blockstore v1.2.0
|
||||
github.com/ipfs/go-ipfs-ds-help v1.1.0
|
||||
github.com/ipfs/go-ipld-format v0.4.0
|
||||
github.com/jackc/pgconn v1.10.0
|
||||
github.com/jackc/pgx/v4 v4.13.0
|
||||
github.com/jackpal/go-nat-pmp v1.0.2
|
||||
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
@ -58,7 +66,7 @@ require (
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
|
||||
github.com/tyler-smith/go-bip39 v1.1.0
|
||||
github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa
|
||||
golang.org/x/crypto v0.1.0
|
||||
golang.org/x/crypto v0.6.0
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
|
||||
golang.org/x/sync v0.1.0
|
||||
golang.org/x/sys v0.5.0
|
||||
@ -68,6 +76,15 @@ require (
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/georgysavva/scany v1.2.1
|
||||
github.com/jmoiron/sqlx v1.3.5
|
||||
github.com/lib/pq v1.10.7
|
||||
github.com/multiformats/go-multihash v0.0.15
|
||||
github.com/pganalyze/pg_query_go/v2 v2.2.0
|
||||
github.com/thoas/go-funk v0.9.3
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect
|
||||
@ -90,35 +107,60 @@ require (
|
||||
github.com/deepmap/oapi-codegen v1.8.2 // indirect
|
||||
github.com/dlclark/regexp2 v1.7.0 // indirect
|
||||
github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect
|
||||
github.com/getsentry/sentry-go v0.18.0 // indirect
|
||||
github.com/getsentry/sentry-go v0.17.0 // indirect
|
||||
github.com/go-ole/go-ole v1.2.1 // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
|
||||
github.com/go-sql-driver/mysql v1.7.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/ipfs/bbloom v0.0.4 // indirect
|
||||
github.com/ipfs/go-datastore v0.5.0 // indirect
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
|
||||
github.com/ipfs/go-log v0.0.1 // indirect
|
||||
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
|
||||
github.com/jackc/pgio v1.0.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgproto3/v2 v2.1.1 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
|
||||
github.com/jackc/pgtype v1.8.1 // indirect
|
||||
github.com/jackc/puddle v1.1.3 // indirect
|
||||
github.com/jbenet/goprocess v0.1.4 // indirect
|
||||
github.com/klauspost/compress v1.15.15 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.9 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.16 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.1 // indirect
|
||||
github.com/mitchellh/pointerstructure v1.2.0 // indirect
|
||||
github.com/mmcloughlin/addchain v0.4.0 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
github.com/multiformats/go-base32 v0.0.3 // indirect
|
||||
github.com/multiformats/go-base36 v0.1.0 // indirect
|
||||
github.com/multiformats/go-multibase v0.0.3 // indirect
|
||||
github.com/multiformats/go-varint v0.0.6 // indirect
|
||||
github.com/naoina/go-stringutil v0.1.0 // indirect
|
||||
github.com/opentracing/opentracing-go v1.1.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_golang v1.14.0 // indirect
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.39.0 // indirect
|
||||
github.com/prometheus/common v0.37.1 // indirect
|
||||
github.com/prometheus/procfs v0.9.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.9.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.5 // indirect
|
||||
github.com/tklauser/numcpus v0.2.2 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.11 // indirect
|
||||
github.com/tklauser/numcpus v0.6.0 // indirect
|
||||
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||
go.uber.org/atomic v1.6.0 // indirect
|
||||
golang.org/x/mod v0.6.0 // indirect
|
||||
golang.org/x/net v0.4.0 // indirect
|
||||
golang.org/x/net v0.6.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
||||
432
go.sum
432
go.sum
@ -8,14 +8,31 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
|
||||
cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
|
||||
cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
|
||||
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
|
||||
cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
|
||||
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
@ -33,6 +50,7 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
|
||||
github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
|
||||
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
|
||||
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8=
|
||||
@ -42,7 +60,10 @@ github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNu
|
||||
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
|
||||
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
|
||||
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
|
||||
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
|
||||
@ -82,6 +103,7 @@ github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
|
||||
github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
@ -90,7 +112,12 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/cloudflare-go v0.14.0 h1:gFqGlGl/5f9UGXAaKapCGUfaTCgRKKnzu2VvzMZlOFA=
|
||||
github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
|
||||
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.2.0 h1:/5znzg5n373N/3ESjHF5SMLxiW4RKB05Ql//KWfeTFs=
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.2.0/go.mod h1:u3MiKYGupPPjkn3ozknpMUpxPaNLTFWAya419/zv6eI=
|
||||
github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=
|
||||
github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8=
|
||||
@ -110,12 +137,15 @@ github.com/consensys/gnark-crypto v0.9.1-0.20230105202408-1a7a29904a7c/go.mod h1
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 h1:6IrxszG5G+O7zhtkWxq6+unVvnrm1fqV2Pe+T95DUzw=
|
||||
github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
|
||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
|
||||
github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
|
||||
@ -156,6 +186,7 @@ github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaB
|
||||
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
|
||||
@ -179,11 +210,13 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqG
|
||||
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
|
||||
github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 h1:AB7YjNrzlVHsYz06zCULVV2zYCEft82P86dSmtwxKL0=
|
||||
github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732/go.mod h1:o/XfIXWi4/GqbQirfRm5uTbXMG5NpqxkxblnbZ+QM9I=
|
||||
github.com/georgysavva/scany v1.2.1 h1:91PAMBpwBtDjvn46TaLQmuVhxpAG6p6sjQaU4zPHPSM=
|
||||
github.com/georgysavva/scany v1.2.1/go.mod h1:vGBpL5XRLOocMFFa55pj0P04DrL3I7qKVRL49K6Eu5o=
|
||||
github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
|
||||
github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
|
||||
github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c=
|
||||
github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0=
|
||||
github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ=
|
||||
github.com/getsentry/sentry-go v0.17.0 h1:UustVWnOoDFHBS7IJUB2QK/nB5pap748ZEp0swnQJak=
|
||||
github.com/getsentry/sentry-go v0.17.0/go.mod h1:B82dxtBvxG0KaPD8/hfSV+VcHD+Lg/xUS4JuQn1P4cM=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
|
||||
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
|
||||
@ -195,11 +228,17 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
|
||||
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
|
||||
github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
|
||||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
||||
@ -208,17 +247,27 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
|
||||
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
|
||||
github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
|
||||
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
|
||||
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
|
||||
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
|
||||
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
|
||||
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
@ -231,12 +280,19 @@ github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgR
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
@ -245,6 +301,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
@ -263,6 +320,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
@ -271,11 +329,17 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
|
||||
github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64=
|
||||
github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
@ -288,11 +352,15 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0=
|
||||
github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
|
||||
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
|
||||
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
|
||||
github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
|
||||
github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
|
||||
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e h1:pIYdhNkDh+YENVNi3gto8n9hAmRxKxoar0iE6BLucjw=
|
||||
github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e/go.mod h1:j9cQbcqHQujT0oKJ38PylVfqohClLr3CvDC+Qcg+lhU=
|
||||
@ -322,19 +390,122 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y
|
||||
github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE=
|
||||
github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0=
|
||||
github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
|
||||
github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc=
|
||||
github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
|
||||
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
|
||||
github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
|
||||
github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0=
|
||||
github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro=
|
||||
github.com/ipfs/go-datastore v0.5.0 h1:rQicVCEacWyk4JZ6G5bD9TKR7lZEG1MWcG7UdWYrFAU=
|
||||
github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk=
|
||||
github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
|
||||
github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
|
||||
github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw=
|
||||
github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE=
|
||||
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
|
||||
github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q=
|
||||
github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU=
|
||||
github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8=
|
||||
github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
|
||||
github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM=
|
||||
github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ=
|
||||
github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM=
|
||||
github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
|
||||
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
|
||||
github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
|
||||
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
|
||||
github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
|
||||
github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
|
||||
github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
|
||||
github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=
|
||||
github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
|
||||
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
|
||||
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
|
||||
github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
|
||||
github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
|
||||
github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
|
||||
github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
|
||||
github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk=
|
||||
github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
|
||||
github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
|
||||
github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
|
||||
github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
|
||||
github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
|
||||
github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU=
|
||||
github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
|
||||
github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
|
||||
github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
|
||||
github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
|
||||
github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c=
|
||||
github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc=
|
||||
github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
|
||||
github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
|
||||
github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
|
||||
github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
|
||||
github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
|
||||
github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
|
||||
github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
|
||||
github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI=
|
||||
github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
|
||||
github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
|
||||
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
|
||||
github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
|
||||
github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
|
||||
github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
|
||||
github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0=
|
||||
github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
|
||||
github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
|
||||
github.com/jackc/pgtype v1.6.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
|
||||
github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
|
||||
github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs=
|
||||
github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
|
||||
github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
|
||||
github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
|
||||
github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
|
||||
github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA=
|
||||
github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
|
||||
github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
|
||||
github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA=
|
||||
github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
|
||||
github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570=
|
||||
github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0=
|
||||
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94=
|
||||
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
|
||||
github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
|
||||
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
|
||||
github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
|
||||
github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
|
||||
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U=
|
||||
github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
|
||||
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
|
||||
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o=
|
||||
@ -352,6 +523,7 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb
|
||||
github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
|
||||
github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
|
||||
github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
@ -362,16 +534,23 @@ github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7y
|
||||
github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
|
||||
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
|
||||
github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
@ -383,18 +562,28 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
|
||||
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
|
||||
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
|
||||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
|
||||
github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
@ -406,6 +595,9 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
|
||||
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
|
||||
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
@ -413,6 +605,12 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
|
||||
github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
|
||||
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
|
||||
@ -426,10 +624,31 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
|
||||
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
|
||||
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
|
||||
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
|
||||
github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
|
||||
github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
|
||||
github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
|
||||
github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
|
||||
github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
|
||||
github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
|
||||
github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
|
||||
github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
|
||||
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
|
||||
github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
|
||||
github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM=
|
||||
github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
|
||||
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
|
||||
github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
|
||||
github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
|
||||
github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0=
|
||||
@ -460,6 +679,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
|
||||
github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
|
||||
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM=
|
||||
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
|
||||
github.com/pganalyze/pg_query_go/v2 v2.2.0 h1:OW+reH+ZY7jdEuPyuLGlf1m7dLbE+fDudKXhLs0Ttpk=
|
||||
github.com/pganalyze/pg_query_go/v2 v2.2.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA=
|
||||
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
@ -474,20 +695,30 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
|
||||
github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
|
||||
github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
|
||||
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
|
||||
github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI=
|
||||
github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
|
||||
github.com/prometheus/common v0.37.1 h1:pYY6b5sGXqEB0WwcRGAoVGKbxVthy9qF17R4gbHZVe0=
|
||||
github.com/prometheus/common v0.37.1/go.mod h1:jEuMeTn4pKGSAxwr7rXtOD70GeY0ERpt0d9FkKf9sK4=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
|
||||
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
|
||||
github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
|
||||
@ -500,22 +731,34 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
|
||||
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
|
||||
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
|
||||
github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
|
||||
github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
|
||||
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
|
||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
@ -527,6 +770,8 @@ github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobt
|
||||
github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
@ -541,11 +786,13 @@ github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 h1:m+8fKfQwCA
|
||||
github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
|
||||
github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
|
||||
github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
|
||||
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4=
|
||||
github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
|
||||
github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
|
||||
github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
|
||||
github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
|
||||
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
|
||||
github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
|
||||
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
|
||||
github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8=
|
||||
github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
@ -561,6 +808,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn
|
||||
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
|
||||
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
|
||||
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
|
||||
@ -573,31 +822,56 @@ github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmv
|
||||
github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
|
||||
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
|
||||
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
|
||||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
|
||||
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
|
||||
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
|
||||
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
|
||||
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
|
||||
golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@ -608,6 +882,9 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0
|
||||
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
||||
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||
@ -621,15 +898,20 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
|
||||
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@ -639,6 +921,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
@ -647,11 +930,24 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
@ -660,24 +956,34 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
|
||||
golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -687,9 +993,12 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -697,40 +1006,67 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
@ -739,14 +1075,17 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@ -760,27 +1099,53 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
|
||||
golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
||||
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
|
||||
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
|
||||
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@ -800,11 +1165,21 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn
|
||||
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
@ -820,7 +1195,23 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx
|
||||
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
|
||||
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
|
||||
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
@ -830,6 +1221,11 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
|
||||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
@ -839,6 +1235,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
@ -854,6 +1251,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
|
||||
gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
|
||||
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
|
||||
gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
|
||||
@ -864,6 +1262,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
@ -873,12 +1272,19 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg=
|
||||
gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
|
||||
gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
|
||||
rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=
|
||||
|
||||
@ -82,6 +82,9 @@ func (c *collector) addTimer(name string, m metrics.Timer) {
|
||||
c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i])
|
||||
}
|
||||
c.buff.WriteRune('\n')
|
||||
|
||||
c.buff.WriteString(fmt.Sprintf(typeGaugeTpl, mutateKey(name+"_total")))
|
||||
c.buff.WriteString(fmt.Sprintf(keyValueTpl, mutateKey(name+"_total"), m.Total()))
|
||||
}
|
||||
|
||||
func (c *collector) addResettingTimer(name string, m metrics.ResettingTimer) {
|
||||
|
||||
@ -92,6 +92,9 @@ test_timer {quantile="0.99"} 1.2e+08
|
||||
test_timer {quantile="0.999"} 1.2e+08
|
||||
test_timer {quantile="0.9999"} 1.2e+08
|
||||
|
||||
# TYPE test_timer_total gauge
|
||||
test_timer_total 230000000
|
||||
|
||||
# TYPE test_resetting_timer_count counter
|
||||
test_resetting_timer_count 6
|
||||
|
||||
|
||||
@ -25,6 +25,7 @@ type Timer interface {
|
||||
Update(time.Duration)
|
||||
UpdateSince(time.Time)
|
||||
Variance() float64
|
||||
Total() int64
|
||||
}
|
||||
|
||||
// GetOrRegisterTimer returns an existing Timer or constructs and registers a
|
||||
@ -47,6 +48,7 @@ func NewCustomTimer(h Histogram, m Meter) Timer {
|
||||
return &StandardTimer{
|
||||
histogram: h,
|
||||
meter: m,
|
||||
total: NewCounter(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -72,6 +74,7 @@ func NewTimer() Timer {
|
||||
return &StandardTimer{
|
||||
histogram: NewHistogram(NewExpDecaySample(1028, 0.015)),
|
||||
meter: NewMeter(),
|
||||
total: NewCounter(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -134,11 +137,15 @@ func (NilTimer) UpdateSince(time.Time) {}
|
||||
// Variance is a no-op.
|
||||
func (NilTimer) Variance() float64 { return 0.0 }
|
||||
|
||||
// Total is a no-op.
|
||||
func (NilTimer) Total() int64 { return int64(0) }
|
||||
|
||||
// StandardTimer is the standard implementation of a Timer and uses a Histogram
|
||||
// and Meter.
|
||||
type StandardTimer struct {
|
||||
histogram Histogram
|
||||
meter Meter
|
||||
total Counter
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
@ -200,6 +207,7 @@ func (t *StandardTimer) Snapshot() Timer {
|
||||
return &TimerSnapshot{
|
||||
histogram: t.histogram.Snapshot().(*HistogramSnapshot),
|
||||
meter: t.meter.Snapshot().(*MeterSnapshot),
|
||||
total: t.total.Snapshot().(CounterSnapshot),
|
||||
}
|
||||
}
|
||||
|
||||
@ -231,14 +239,12 @@ func (t *StandardTimer) Update(d time.Duration) {
|
||||
defer t.mutex.Unlock()
|
||||
t.histogram.Update(int64(d))
|
||||
t.meter.Mark(1)
|
||||
t.total.Inc(int64(d))
|
||||
}
|
||||
|
||||
// Record the duration of an event that started at a time and ends now.
|
||||
func (t *StandardTimer) UpdateSince(ts time.Time) {
|
||||
t.mutex.Lock()
|
||||
defer t.mutex.Unlock()
|
||||
t.histogram.Update(int64(time.Since(ts)))
|
||||
t.meter.Mark(1)
|
||||
t.Update(time.Since(ts))
|
||||
}
|
||||
|
||||
// Variance returns the variance of the values in the sample.
|
||||
@ -246,10 +252,18 @@ func (t *StandardTimer) Variance() float64 {
|
||||
return t.histogram.Variance()
|
||||
}
|
||||
|
||||
// Total returns the total time the timer has run in nanoseconds.
|
||||
// This differs from Sum in that it is a simple counter, not based
|
||||
// on a histogram Sample.
|
||||
func (t *StandardTimer) Total() int64 {
|
||||
return t.total.Count()
|
||||
}
|
||||
|
||||
// TimerSnapshot is a read-only copy of another Timer.
|
||||
type TimerSnapshot struct {
|
||||
histogram *HistogramSnapshot
|
||||
meter *MeterSnapshot
|
||||
total CounterSnapshot
|
||||
}
|
||||
|
||||
// Count returns the number of events recorded at the time the snapshot was
|
||||
@ -324,3 +338,6 @@ func (*TimerSnapshot) UpdateSince(time.Time) {
|
||||
// Variance returns the variance of the values at the time the snapshot was
|
||||
// taken.
|
||||
func (t *TimerSnapshot) Variance() float64 { return t.histogram.Variance() }
|
||||
|
||||
// Total returns the total time the timer has run in nanoseconds.
|
||||
func (t *TimerSnapshot) Total() int64 { return t.total.Count() }
|
||||
|
||||
262
mobile/android_test.go
Normal file
262
mobile/android_test.go
Normal file
@ -0,0 +1,262 @@
|
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package geth
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cespare/cp"
|
||||
)
|
||||
|
||||
// androidTestClass is a Java class to do some lightweight tests against the Android
|
||||
// bindings. The goal is not to test each individual functionality, rather just to
|
||||
// catch breaking API and/or implementation changes.
|
||||
const androidTestClass = `
|
||||
package go;
|
||||
|
||||
import android.test.InstrumentationTestCase;
|
||||
import android.test.MoreAsserts;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.ethereum.geth.*;
|
||||
|
||||
public class AndroidTest extends InstrumentationTestCase {
|
||||
public AndroidTest() {}
|
||||
|
||||
public void testAccountManagement() {
|
||||
// Create an encrypted keystore with light crypto parameters.
|
||||
KeyStore ks = new KeyStore(getInstrumentation().getContext().getFilesDir() + "/keystore", Geth.LightScryptN, Geth.LightScryptP);
|
||||
|
||||
try {
|
||||
// Create a new account with the specified encryption passphrase.
|
||||
Account newAcc = ks.newAccount("Creation password");
|
||||
|
||||
// Export the newly created account with a different passphrase. The returned
|
||||
// data from this method invocation is a JSON encoded, encrypted key-file.
|
||||
byte[] jsonAcc = ks.exportKey(newAcc, "Creation password", "Export password");
|
||||
|
||||
// Update the passphrase on the account created above inside the local keystore.
|
||||
ks.updateAccount(newAcc, "Creation password", "Update password");
|
||||
|
||||
// Delete the account updated above from the local keystore.
|
||||
ks.deleteAccount(newAcc, "Update password");
|
||||
|
||||
// Import back the account we've exported (and then deleted) above with yet
|
||||
// again a fresh passphrase.
|
||||
Account impAcc = ks.importKey(jsonAcc, "Export password", "Import password");
|
||||
|
||||
// Create a new account to sign transactions with
|
||||
Account signer = ks.newAccount("Signer password");
|
||||
|
||||
Transaction tx = new Transaction(
|
||||
1, new Address("0x0000000000000000000000000000000000000000"),
|
||||
new BigInt(0), 0, new BigInt(1), null); // Random empty transaction
|
||||
BigInt chain = new BigInt(1); // Chain identifier of the main net
|
||||
|
||||
// Sign a transaction with a single authorization
|
||||
Transaction signed = ks.signTxPassphrase(signer, "Signer password", tx, chain);
|
||||
|
||||
// Sign a transaction with multiple manually cancelled authorizations
|
||||
ks.unlock(signer, "Signer password");
|
||||
signed = ks.signTx(signer, tx, chain);
|
||||
ks.lock(signer.getAddress());
|
||||
|
||||
// Sign a transaction with multiple automatically cancelled authorizations
|
||||
ks.timedUnlock(signer, "Signer password", 1000000000);
|
||||
signed = ks.signTx(signer, tx, chain);
|
||||
} catch (Exception e) {
|
||||
fail(e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
public void testInprocNode() {
|
||||
Context ctx = new Context();
|
||||
|
||||
try {
|
||||
// Start up a new inprocess node
|
||||
Node node = new Node(getInstrumentation().getContext().getFilesDir() + "/.ethereum", new NodeConfig());
|
||||
node.start();
|
||||
|
||||
// Retrieve some data via function calls (we don't really care about the results)
|
||||
NodeInfo info = node.getNodeInfo();
|
||||
info.getName();
|
||||
info.getListenerAddress();
|
||||
info.getProtocols();
|
||||
|
||||
// Retrieve some data via the APIs (we don't really care about the results)
|
||||
EthereumClient ec = node.getEthereumClient();
|
||||
ec.getBlockByNumber(ctx, -1).getNumber();
|
||||
|
||||
NewHeadHandler handler = new NewHeadHandler() {
|
||||
@Override public void onError(String error) {}
|
||||
@Override public void onNewHead(final Header header) {}
|
||||
};
|
||||
ec.subscribeNewHead(ctx, handler, 16);
|
||||
} catch (Exception e) {
|
||||
fail(e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that recovering transaction signers works for both Homestead and EIP155
|
||||
// signatures too. Regression test for go-ethereum issue #14599.
|
||||
public void testIssue14599() {
|
||||
try {
|
||||
byte[] preEIP155RLP = new BigInteger("f901fc8032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b561ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884", 16).toByteArray();
|
||||
preEIP155RLP = Arrays.copyOfRange(preEIP155RLP, 1, preEIP155RLP.length);
|
||||
|
||||
byte[] postEIP155RLP = new BigInteger("f86b80847735940082520894ef5bbb9bba2e1ca69ef81b23a8727d889f3ef0a1880de0b6b3a7640000802ba06fef16c44726a102e6d55a651740636ef8aec6df3ebf009e7b0c1f29e4ac114aa057e7fbc69760b522a78bb568cfc37a58bfdcf6ea86cb8f9b550263f58074b9cc", 16).toByteArray();
|
||||
postEIP155RLP = Arrays.copyOfRange(postEIP155RLP, 1, postEIP155RLP.length);
|
||||
|
||||
Transaction preEIP155 = new Transaction(preEIP155RLP);
|
||||
Transaction postEIP155 = new Transaction(postEIP155RLP);
|
||||
|
||||
preEIP155.getFrom(null); // Homestead should accept homestead
|
||||
preEIP155.getFrom(new BigInt(4)); // EIP155 should accept homestead (missing chain ID)
|
||||
postEIP155.getFrom(new BigInt(4)); // EIP155 should accept EIP 155
|
||||
|
||||
try {
|
||||
postEIP155.getFrom(null);
|
||||
fail("EIP155 transaction accepted by Homestead");
|
||||
} catch (Exception e) {}
|
||||
} catch (Exception e) {
|
||||
fail(e.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
// TestAndroid runs the Android java test class specified above.
|
||||
//
|
||||
// This requires the gradle command in PATH and the Android SDK whose path is available
|
||||
// through ANDROID_HOME environment variable. To successfully run the tests, an Android
|
||||
// device must also be available with debugging enabled.
|
||||
//
|
||||
// This method has been adapted from golang.org/x/mobile/bind/java/seq_test.go/runTest
|
||||
func TestAndroid(t *testing.T) {
|
||||
t.Skip("Skipping this test for statediff as this is not relevant")
|
||||
// Skip tests on Windows altogether
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("cannot test Android bindings on Windows, skipping")
|
||||
}
|
||||
// Make sure all the Android tools are installed
|
||||
if _, err := exec.Command("which", "gradle").CombinedOutput(); err != nil {
|
||||
t.Skip("command gradle not found, skipping")
|
||||
}
|
||||
if sdk := os.Getenv("ANDROID_HOME"); sdk == "" {
|
||||
// Android SDK not explicitly given, try to auto-resolve
|
||||
autopath := filepath.Join(os.Getenv("HOME"), "Android", "Sdk")
|
||||
if _, err := os.Stat(autopath); err != nil {
|
||||
t.Skip("ANDROID_HOME environment var not set, skipping")
|
||||
}
|
||||
os.Setenv("ANDROID_HOME", autopath)
|
||||
}
|
||||
if _, err := exec.Command("which", "gomobile").CombinedOutput(); err != nil {
|
||||
t.Log("gomobile missing, installing it...")
|
||||
if out, err := exec.Command("go", "get", "golang.org/x/mobile/cmd/gomobile").CombinedOutput(); err != nil {
|
||||
t.Fatalf("install failed: %v\n%s", err, string(out))
|
||||
}
|
||||
t.Log("initializing gomobile...")
|
||||
start := time.Now()
|
||||
if _, err := exec.Command("gomobile", "init").CombinedOutput(); err != nil {
|
||||
t.Fatalf("initialization failed: %v", err)
|
||||
}
|
||||
t.Logf("initialization took %v", time.Since(start))
|
||||
}
|
||||
// Create and switch to a temporary workspace
|
||||
workspace := t.TempDir()
|
||||
|
||||
pwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get current working directory: %v", err)
|
||||
}
|
||||
if err := os.Chdir(workspace); err != nil {
|
||||
t.Fatalf("failed to switch to temporary workspace: %v", err)
|
||||
}
|
||||
defer os.Chdir(pwd)
|
||||
|
||||
// Create the skeleton of the Android project
|
||||
for _, dir := range []string{"src/main", "src/androidTest/java/org/ethereum/gethtest", "libs"} {
|
||||
err = os.MkdirAll(dir, os.ModePerm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Generate the mobile bindings for Geth and add the tester class
|
||||
gobind := exec.Command("gomobile", "bind", "-javapkg", "org.ethereum", "github.com/ethereum/go-ethereum/mobile")
|
||||
if output, err := gobind.CombinedOutput(); err != nil {
|
||||
t.Logf("%s", output)
|
||||
t.Fatalf("failed to run gomobile bind: %v", err)
|
||||
}
|
||||
cp.CopyFile(filepath.Join("libs", "geth.aar"), "geth.aar")
|
||||
|
||||
if err = os.WriteFile(filepath.Join("src", "androidTest", "java", "org", "ethereum", "gethtest", "AndroidTest.java"), []byte(androidTestClass), os.ModePerm); err != nil {
|
||||
t.Fatalf("failed to write Android test class: %v", err)
|
||||
}
|
||||
// Finish creating the project and run the tests via gradle
|
||||
if err = os.WriteFile(filepath.Join("src", "main", "AndroidManifest.xml"), []byte(androidManifest), os.ModePerm); err != nil {
|
||||
t.Fatalf("failed to write Android manifest: %v", err)
|
||||
}
|
||||
if err = os.WriteFile("build.gradle", []byte(gradleConfig), os.ModePerm); err != nil {
|
||||
t.Fatalf("failed to write gradle build file: %v", err)
|
||||
}
|
||||
if output, err := exec.Command("gradle", "connectedAndroidTest").CombinedOutput(); err != nil {
|
||||
t.Logf("%s", output)
|
||||
t.Errorf("failed to run gradle test: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
const androidManifest = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="org.ethereum.gethtest"
|
||||
android:versionCode="1"
|
||||
android:versionName="1.0">
|
||||
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
</manifest>`
|
||||
|
||||
const gradleConfig = `buildscript {
|
||||
repositories {
|
||||
jcenter()
|
||||
}
|
||||
dependencies {
|
||||
classpath 'com.android.tools.build:gradle:2.2.3'
|
||||
}
|
||||
}
|
||||
allprojects {
|
||||
repositories { jcenter() }
|
||||
}
|
||||
apply plugin: 'com.android.library'
|
||||
android {
|
||||
compileSdkVersion 'android-19'
|
||||
buildToolsVersion '21.1.2'
|
||||
defaultConfig { minSdkVersion 15 }
|
||||
}
|
||||
repositories {
|
||||
flatDir { dirs 'libs' }
|
||||
}
|
||||
dependencies {
|
||||
compile 'com.android.support:appcompat-v7:19.0.0'
|
||||
compile(name: "geth", ext: "aar")
|
||||
}
|
||||
`
|
||||
@ -440,6 +440,9 @@ type ChainConfig struct {
|
||||
// the network that triggers the consensus upgrade.
|
||||
TerminalTotalDifficulty *big.Int `json:"terminalTotalDifficulty,omitempty"`
|
||||
|
||||
// Cap the maximum total difficulty (for testnet use only).
|
||||
CappedMaximumDifficulty *big.Int `json:"cappedMaximumDifficulty,omitempty"`
|
||||
|
||||
// TerminalTotalDifficultyPassed is a flag specifying that the network already
|
||||
// passed the terminal total difficulty. Its purpose is to disable legacy sync
|
||||
// even without having seen the TTD locally (safer long term).
|
||||
@ -482,7 +485,11 @@ func (c *ChainConfig) Description() string {
|
||||
switch {
|
||||
case c.Ethash != nil:
|
||||
if c.TerminalTotalDifficulty == nil {
|
||||
banner += "Consensus: Ethash (proof-of-work)\n"
|
||||
if nil == c.CappedMaximumDifficulty {
|
||||
banner += "Consensus: Ethash (proof-of-work)\n"
|
||||
} else {
|
||||
banner += fmt.Sprintf("Consensus: Ethash (proof-of-work, capped difficulty at %d)\n", c.CappedMaximumDifficulty)
|
||||
}
|
||||
} else if !c.TerminalTotalDifficultyPassed {
|
||||
banner += "Consensus: Beacon (proof-of-stake), merging from Ethash (proof-of-work)\n"
|
||||
} else {
|
||||
@ -497,7 +504,11 @@ func (c *ChainConfig) Description() string {
|
||||
banner += "Consensus: Beacon (proof-of-stake), merged from Clique (proof-of-authority)\n"
|
||||
}
|
||||
default:
|
||||
banner += "Consensus: unknown\n"
|
||||
if nil == c.CappedMaximumDifficulty {
|
||||
banner += "Consensus: unknown\n"
|
||||
} else {
|
||||
banner += fmt.Sprintf("Consensus: unknown (capped difficulty at %d)\n", c.CappedMaximumDifficulty)
|
||||
}
|
||||
}
|
||||
banner += "\n"
|
||||
|
||||
|
||||
@ -21,10 +21,10 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
VersionMajor = 1 // Major version component of the current release
|
||||
VersionMinor = 11 // Minor version component of the current release
|
||||
VersionPatch = 2 // Patch version component of the current release
|
||||
VersionMeta = "stable" // Version metadata to append to the version string
|
||||
VersionMajor = 1 // Major version component of the current release
|
||||
VersionMinor = 11 // Minor version component of the current release
|
||||
VersionPatch = 2 // Patch version component of the current release
|
||||
VersionMeta = "statediff-4.3.7-alpha" // Version metadata to append to the version string
|
||||
)
|
||||
|
||||
// Version holds the textual version string.
|
||||
|
||||
@ -33,7 +33,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
maxRequestContentLength = 1024 * 1024 * 5
|
||||
maxRequestContentLength = 1024 * 1024 * 12
|
||||
contentType = "application/json"
|
||||
)
|
||||
|
||||
|
||||
25
scripts/run_unit_test.sh
Executable file
25
scripts/run_unit_test.sh
Executable file
@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
mkdir -p out
|
||||
|
||||
# Remove existing docker-tsdb directory
|
||||
rm -rf out/docker-tsdb/
|
||||
|
||||
# Copy over files to setup TimescaleDB
|
||||
ID=$(docker create vulcanize/ipld-eth-db:v4.1.1-alpha)
|
||||
docker cp $ID:/app/docker-tsdb out/docker-tsdb/
|
||||
docker rm -v $ID
|
||||
|
||||
# Spin up TimescaleDB
|
||||
docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml up ipld-eth-db
|
||||
sleep 45
|
||||
|
||||
# Run unit tests
|
||||
go clean -testcache
|
||||
make statedifftest
|
||||
|
||||
# Clean up
|
||||
docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes
|
||||
rm -rf out/docker-tsdb/
|
||||
320
statediff/README.md
Normal file
320
statediff/README.md
Normal file
@ -0,0 +1,320 @@
|
||||
# Statediff
|
||||
|
||||
This package provides an auxiliary service that asynchronously processes state diff objects from chain events,
|
||||
either relaying the state objects to RPC subscribers or writing them directly to Postgres as IPLD objects.
|
||||
|
||||
It also exposes RPC endpoints for fetching or writing to Postgres the state diff at a specific block height
|
||||
or for a specific block hash, this operates on historical block and state data and so depends on a complete state archive.
|
||||
|
||||
Data is emitted in this differential format in order to make it feasible to IPLD-ize and index the _entire_ Ethereum state
|
||||
(including intermediate state and storage trie nodes). If this state diff process is ran continuously from genesis,
|
||||
the entire state at any block can be materialized from the cumulative differentials up to that point.
|
||||
|
||||
## Statediff object
|
||||
|
||||
A state diff `StateObject` is the collection of all the state and storage trie nodes that have been updated in a given block.
|
||||
For convenience, we also associate these nodes with the block number and hash, and optionally the set of code hashes and code for any
|
||||
contracts deployed in this block.
|
||||
|
||||
A complete state diff `StateObject` will include all state and storage intermediate nodes, which is necessary for generating proofs and for
|
||||
traversing the tries.
|
||||
|
||||
```go
|
||||
// StateObject is a collection of state (and linked storage nodes) as well as the associated block number, block hash,
|
||||
// and a set of code hashes and their code
|
||||
type StateObject struct {
|
||||
BlockNumber *big.Int `json:"blockNumber" gencodec:"required"`
|
||||
BlockHash common.Hash `json:"blockHash" gencodec:"required"`
|
||||
Nodes []StateNode `json:"nodes" gencodec:"required"`
|
||||
CodeAndCodeHashes []CodeAndCodeHash `json:"codeMapping"`
|
||||
}
|
||||
|
||||
// StateNode holds the data for a single state diff node
|
||||
type StateNode struct {
|
||||
NodeType NodeType `json:"nodeType" gencodec:"required"`
|
||||
Path []byte `json:"path" gencodec:"required"`
|
||||
NodeValue []byte `json:"value" gencodec:"required"`
|
||||
StorageNodes []StorageNode `json:"storage"`
|
||||
LeafKey []byte `json:"leafKey"`
|
||||
}
|
||||
|
||||
// StorageNode holds the data for a single storage diff node
|
||||
type StorageNode struct {
|
||||
NodeType NodeType `json:"nodeType" gencodec:"required"`
|
||||
Path []byte `json:"path" gencodec:"required"`
|
||||
NodeValue []byte `json:"value" gencodec:"required"`
|
||||
LeafKey []byte `json:"leafKey"`
|
||||
}
|
||||
|
||||
// CodeAndCodeHash struct for holding codehash => code mappings
|
||||
// we can't use an actual map because they are not rlp serializable
|
||||
type CodeAndCodeHash struct {
|
||||
Hash common.Hash `json:"codeHash"`
|
||||
Code []byte `json:"code"`
|
||||
}
|
||||
```
|
||||
|
||||
These objects are packed into a `Payload` structure which can additionally associate the `StateObject`
|
||||
with the block (header, uncles, and transactions), receipts, and total difficulty.
|
||||
This `Payload` encapsulates all of the differential data at a given block, and allows us to index the entire Ethereum data structure
|
||||
as hash-linked IPLD objects.
|
||||
|
||||
```go
|
||||
// Payload packages the data to send to state diff subscriptions
|
||||
type Payload struct {
|
||||
BlockRlp []byte `json:"blockRlp"`
|
||||
TotalDifficulty *big.Int `json:"totalDifficulty"`
|
||||
ReceiptsRlp []byte `json:"receiptsRlp"`
|
||||
StateObjectRlp []byte `json:"stateObjectRlp" gencodec:"required"`
|
||||
|
||||
encoded []byte
|
||||
err error
|
||||
}
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
This state diffing service runs as an auxiliary service concurrent to the regular syncing process of the geth node.
|
||||
|
||||
### CLI configuration
|
||||
|
||||
This service introduces a CLI flag namespace `statediff`
|
||||
|
||||
`--statediff` flag is used to turn on the service
|
||||
|
||||
`--statediff.writing` is used to tell the service to write state diff objects it produces from synced ChainEvents directly to a configured Postgres database
|
||||
|
||||
`--statediff.workers` is used to set the number of concurrent workers to process state diff objects and write them into the database
|
||||
|
||||
`--statediff.db.type` is the type of database we write out to (current options: postgres, dump, file)
|
||||
|
||||
`--statediff.dump.dst` is the destination to write to when operating in database dump mode (stdout, stderr, discard)
|
||||
|
||||
`--statediff.db.driver` is the specific driver to use for the database (current options for postgres: pgx and sqlx)
|
||||
|
||||
`--statediff.db.host` is the hostname/ip to dial to connect to the database
|
||||
|
||||
`--statediff.db.port` is the port to dial to connect to the database
|
||||
|
||||
`--statediff.db.name` is the name of the database to connect to
|
||||
|
||||
`--statediff.db.user` is the user to connect to the database as
|
||||
|
||||
`--statediff.db.password` is the password to use to connect to the database
|
||||
|
||||
`--statediff.db.conntimeout` is the connection timeout (in seconds)
|
||||
|
||||
`--statediff.db.maxconns` is the maximum number of database connections
|
||||
|
||||
`--statediff.db.minconns` is the minimum number of database connections
|
||||
|
||||
`--statediff.db.maxidleconns` is the maximum number of idle connections
|
||||
|
||||
`--statediff.db.maxconnidletime` is the maximum lifetime for an idle connection (in seconds)
|
||||
|
||||
`--statediff.db.maxconnlifetime` is the maximum lifetime for a connection (in seconds)
|
||||
|
||||
`--statediff.db.nodeid` is the node id to use in the Postgres database
|
||||
|
||||
`--statediff.db.clientname` is the client name to use in the Postgres database
|
||||
|
||||
`--statediff.db.upsert` whether or not the service, when operating in a direct database writing mode, should overwrite any existing conflicting data
|
||||
|
||||
`--statediff.file.path` full path (including filename) to write statediff data out to when operating in file mode
|
||||
|
||||
`--statediff.file.wapath` full path (including filename) to write statediff watched addresses out to when operating in file mode
|
||||
|
||||
The service can only operate in full sync mode (`--syncmode=full`), but only the historical RPC endpoints require an archive node (`--gcmode=archive`)
|
||||
|
||||
e.g.
|
||||
`./build/bin/geth --syncmode=full --gcmode=archive --statediff --statediff.writing --statediff.db.type=postgres --statediff.db.driver=sqlx --statediff.db.host=localhost --statediff.db.port=5432 --statediff.db.name=vulcanize_test --statediff.db.user=postgres --statediff.db.nodeid=nodeid --statediff.db.clientname=clientname`
|
||||
|
||||
When operating in `--statediff.db.type=file` mode, the service will write SQL statements out to the file designated by
|
||||
`--statediff.file.path`. Please note that it writes out SQL statements with all `ON CONFLICT` constraint checks dropped.
|
||||
This is done so that we can scale out the production of the SQL statements horizontally, merge the separate SQL files produced,
|
||||
de-duplicate using unix tools (`sort statediff.sql | uniq` or `sort -u statediff.sql`), bulk load using psql
|
||||
(`psql db_name --set ON_ERROR_STOP=on -f statediff.sql`), and then add our primary and foreign key constraints and indexes
|
||||
back afterwards.
|
||||
|
||||
### RPC endpoints
|
||||
|
||||
The state diffing service exposes both a WS subscription endpoint, and a number of HTTP unary endpoints.
|
||||
|
||||
Each of these endpoints requires a set of parameters provided by the caller
|
||||
|
||||
```go
|
||||
// Params is used to carry in parameters from subscribing/requesting clients configuration
|
||||
type Params struct {
|
||||
IntermediateStateNodes bool
|
||||
IntermediateStorageNodes bool
|
||||
IncludeBlock bool
|
||||
IncludeReceipts bool
|
||||
IncludeTD bool
|
||||
IncludeCode bool
|
||||
WatchedAddresses []common.Address
|
||||
}
|
||||
```
|
||||
|
||||
Using these params we can tell the service whether to include state and/or storage intermediate nodes; whether
|
||||
to include the associated block (header, uncles, and transactions); whether to include the associated receipts;
|
||||
whether to include the total difficulty for this block; whether to include the set of code hashes and code for
|
||||
contracts deployed in this block; whether to limit the diffing process to a list of specific addresses.
|
||||
|
||||
#### Subscription endpoint
|
||||
|
||||
A websocket supporting RPC endpoint is exposed for subscribing to state diff `StateObjects` that come off the head of the chain while the geth node syncs.
|
||||
|
||||
```go
|
||||
// Stream is a subscription endpoint that fires off state diff payloads as they are created
|
||||
Stream(ctx context.Context, params Params) (*rpc.Subscription, error)
|
||||
```
|
||||
|
||||
To expose this endpoint the node needs to have the websocket server turned on (`--ws`),
|
||||
and the `statediff` namespace exposed (`--ws.api=statediff`).
|
||||
|
||||
Go code subscriptions to this endpoint can be created using the `rpc.Client.Subscribe()` method,
|
||||
with the "statediff" namespace, a `statediff.Payload` channel, and the name of the statediff api's rpc method: "stream".
|
||||
|
||||
e.g.
|
||||
|
||||
```go
|
||||
|
||||
cli, err := rpc.Dial("ipcPathOrWsURL")
|
||||
if err != nil {
|
||||
// handle error
|
||||
}
|
||||
stateDiffPayloadChan := make(chan statediff.Payload, 20000)
|
||||
methodName := "stream"
|
||||
params := statediff.Params{
|
||||
IncludeBlock: true,
|
||||
IncludeTD: true,
|
||||
IncludeReceipts: true,
|
||||
IntermediateStorageNodes: true,
|
||||
IntermediateStateNodes: true,
|
||||
}
|
||||
rpcSub, err := cli.Subscribe(context.Background(), statediff.APIName, stateDiffPayloadChan, methodName, params)
|
||||
if err != nil {
|
||||
// handle error
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case stateDiffPayload := <- stateDiffPayloadChan:
|
||||
// process the payload
|
||||
case err := <- rpcSub.Err():
|
||||
// handle rpc subscription error
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Unary endpoints
|
||||
|
||||
The service also exposes unary RPC endpoints for retrieving the state diff `StateObject` for a specific block height/hash.
|
||||
|
||||
```go
|
||||
// StateDiffAt returns a state diff payload at the specific blockheight
|
||||
StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error)
|
||||
|
||||
// StateDiffFor returns a state diff payload for the specific blockhash
|
||||
StateDiffFor(ctx context.Context, blockHash common.Hash, params Params) (*Payload, error)
|
||||
```
|
||||
|
||||
To expose this endpoint the node needs to have the HTTP server turned on (`--http`),
|
||||
and the `statediff` namespace exposed (`--http.api=statediff`).
|
||||
|
||||
### Direct indexing into Postgres
|
||||
|
||||
If `--statediff.writing` is set, the service will convert the state diff `StateObject` data into IPLD objects, persist them directly to Postgres,
|
||||
and generate secondary indexes around the IPLD data.
|
||||
|
||||
The schema and migrations for this Postgres database are provided in `statediff/db/`.
|
||||
|
||||
#### Postgres setup
|
||||
|
||||
We use [pressly/goose](https://github.com/pressly/goose) as our Postgres migration manager.
|
||||
You can also load the Postgres schema directly into a database using
|
||||
|
||||
`psql database_name < schema.sql`
|
||||
|
||||
This will only work on a version 12.4 Postgres database.
|
||||
|
||||
#### Schema overview
|
||||
|
||||
Our Postgres schemas are built around a single IPFS backing Postgres IPLD blockstore table (`public.blocks`) that conforms with [go-ds-sql](https://github.com/ipfs/go-ds-sql/blob/master/postgres/postgres.go).
|
||||
All IPLD objects are stored in this table, where `key` is the blockstore-prefixed multihash key for the IPLD object and `data` contains
|
||||
the bytes for the IPLD block (in the case of all Ethereum IPLDs, this is the RLP byte encoding of the Ethereum object).
|
||||
|
||||
The IPLD objects in this table can be traversed using an IPLD DAG interface, but since this table only maps multihash to raw IPLD object
|
||||
it is not particularly useful for searching through the data by looking up Ethereum objects by their constituent fields
|
||||
(e.g. by block number, tx source/recipient, state/storage trie node path). To improve the accessibility of these objects
|
||||
we create an Ethereum [advanced data layout](https://github.com/ipld/specs#schemas-and-advanced-data-layouts) (ADL) by generating secondary
|
||||
indexes on top of the raw IPLDs in other Postgres tables.
|
||||
|
||||
These secondary index tables fall under the `eth` schema and follow an `{objectType}_cids` naming convention.
|
||||
These tables provide a view into individual fields of the underlying Ethereum IPLD objects, allowing lookups on these fields, and reference the raw IPLD objects stored in `public.blocks`
|
||||
by foreign keys to their multihash keys.
|
||||
Additionally, these tables maintain the hash-linked nature of Ethereum objects to one another. E.g. a storage trie node entry in the `storage_cids`
|
||||
table contains a `state_id` foreign key which references the `id` for the `state_cids` entry that contains the state leaf node for the contract that storage node belongs to,
|
||||
and in turn that `state_cids` entry contains a `header_id` foreign key which references the `id` of the `header_cids` entry that contains the header for the block these state and storage nodes were updated (diffed).
|
||||
|
||||
### Optimization
|
||||
|
||||
On mainnet this process is extremely IO intensive and requires significant resources to allow it to keep up with the head of the chain.
|
||||
The state diff processing time for a specific block is dependent on the number and complexity of the state changes that occur in a block and
|
||||
the number of updated state nodes that are available in the in-memory cache vs must be retrieved from disc.
|
||||
|
||||
If memory permits, one means of improving the efficiency of this process is to increase the in-memory trie cache allocation.
|
||||
This can be done by increasing the overall `--cache` allocation and/or by increasing the % of the cache allocated to trie
|
||||
usage with `--cache.trie`.
|
||||
|
||||
## Versioning, Branches, Rebasing, and Releasing
|
||||
|
||||
Internal tagged releases are maintained for building the latest version of statediffing geth or using it as a go mod dependency.
|
||||
When a new core go-ethereum version is released, statediffing geth is rebased onto and adjusted to work with the new tag.
|
||||
|
||||
We want to maintain a complete record of our git history, but in order to make frequent and timely rebases feasible we also
|
||||
need to be able to squash our work before performing a rebase. To this end we retain multiple branches with partial incremental history that culminate in
|
||||
the full incremental history.
|
||||
|
||||
### Versioning
|
||||
|
||||
Example: `v1.10.16-statediff-3.0.2`
|
||||
|
||||
- The first section, `v1.10.16`, corresponds to the release of the root branch this version is rebased onto (e.g., [](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.16)[https://github.com/ethereum/go-ethereum/releases/tag/v1.10.16](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.16))
|
||||
- The second section, `3.0.2`, corresponds to the version of our statediffing code. The major version here (3) should always correspond with the major version of the `ipld-eth-db` schema version it works with (e.g., [](https://github.com/vulcanize/ipld-eth-db/releases/tag/v3.0.6)[https://github.com/vulcanize/ipld-eth-db/releases/tag/v3.0.6](https://github.com/vulcanize/ipld-eth-db/releases/tag/v3.0.6)); it is only bumped when we bump the major version of the schema.
|
||||
- The major version of the schema is only bumped when a breaking change is made to the schema.
|
||||
- The minor version is bumped when a new feature is added, or a fix is performed that breaks or updates the statediffing API or CLI in some way.
|
||||
- The patch version is bumped whenever minor fixes/patches/features are done that don’t change/break API/CLI compatibility.
|
||||
- We are very strict about the first section and the major version of the statediffing code, but some discretion is required when deciding to bump minor versus patch version of the statediffing code.
|
||||
|
||||
The statediff version is included in the `VersionMeta` in params/version.go
|
||||
|
||||
### Branches
|
||||
|
||||
We maintain two official kinds of branches:
|
||||
|
||||
Major Branch: `{Root Version}-statediff`
|
||||
Major branches retain the cumulative state of all changes made before the latest root version rebase and track the full incremental history of changes made between the latest root version rebase and the next.
|
||||
Aside from creating the branch by performing the rebase described in the section below, these branches are never worked off of or committed to directly.
|
||||
|
||||
Feature Branch: `{Root Version}-statediff-{Statediff Version}`
|
||||
Feature branches are checked out from a major branch in order to work on a new feature or fix for the statediffing code.
|
||||
The statediff version of a feature branch is the new version it affects on the major branch when merged. Internal tagged releases
|
||||
are cut against these branches after they are merged back to the major branch.
|
||||
|
||||
If a developer is unsure what version their patch should affect, they should remain working on an unofficial branch. From there
|
||||
they can open a PR against the targeted root branch and be directed to the appropriate feature version and branch.
|
||||
|
||||
### Rebasing
|
||||
|
||||
When a new root tagged release comes out we rebase our statediffing code on top of the new tag using the following process:
|
||||
|
||||
1. Checkout a new major branch for the tag from the current major branch
|
||||
2. On the new major branch, squash all our commits since the last major rebase
|
||||
3. On the new major branch, perform the rebase against the new tag
|
||||
4. Push the new major branch to the remote
|
||||
5. From the new major branch, checkout a new feature branch based on the new major version and the last statediff version
|
||||
6. On this new feature branch, add the new major branch to the .github/workflows/on-master.yml list of "on push" branches
|
||||
7. On this new feature branch, make any fixes/adjustments required for all statediffing geth tests to pass
|
||||
8. PR this feature branch into the new major branch, this PR will trigger CI tests and builds.
|
||||
9. After merging PR, rebase feature branch onto major branch
|
||||
10. Cut a new release targeting the feature branch, this release should have the new root version but the same statediff version as the last release
|
||||
213
statediff/api.go
Normal file
213
statediff/api.go
Normal file
@ -0,0 +1,213 @@
|
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package statediff
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/types"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
)
|
||||
|
||||
// APIName is the namespace used for the state diffing service API
|
||||
const APIName = "statediff"
|
||||
|
||||
// APIVersion is the version of the state diffing service API
|
||||
const APIVersion = "0.0.1"
|
||||
|
||||
// PublicStateDiffAPI provides an RPC subscription interface
|
||||
// that can be used to stream out state diffs as they
|
||||
// are produced by a full node
|
||||
type PublicStateDiffAPI struct {
|
||||
sds IService
|
||||
}
|
||||
|
||||
// NewPublicStateDiffAPI creates an rpc subscription interface for the underlying statediff service
|
||||
func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI {
|
||||
return &PublicStateDiffAPI{
|
||||
sds: sds,
|
||||
}
|
||||
}
|
||||
|
||||
// Stream is the public method to setup a subscription that fires off statediff service payloads as they are created
|
||||
func (api *PublicStateDiffAPI) Stream(ctx context.Context, params Params) (*rpc.Subscription, error) {
|
||||
// ensure that the RPC connection supports subscriptions
|
||||
notifier, supported := rpc.NotifierFromContext(ctx)
|
||||
if !supported {
|
||||
return nil, rpc.ErrNotificationsUnsupported
|
||||
}
|
||||
|
||||
// create subscription and start waiting for events
|
||||
rpcSub := notifier.CreateSubscription()
|
||||
|
||||
go func() {
|
||||
// subscribe to events from the statediff service
|
||||
payloadChannel := make(chan Payload, chainEventChanSize)
|
||||
quitChan := make(chan bool, 1)
|
||||
api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan, params)
|
||||
// loop and await payloads and relay them to the subscriber with the notifier
|
||||
for {
|
||||
select {
|
||||
case payload := <-payloadChannel:
|
||||
if err := notifier.Notify(rpcSub.ID, payload); err != nil {
|
||||
log.Error("Failed to send state diff packet; error: " + err.Error())
|
||||
if err := api.sds.Unsubscribe(rpcSub.ID); err != nil {
|
||||
log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
case err := <-rpcSub.Err():
|
||||
if err != nil {
|
||||
log.Error("State diff service rpcSub error: " + err.Error())
|
||||
err = api.sds.Unsubscribe(rpcSub.ID)
|
||||
if err != nil {
|
||||
log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
case <-quitChan:
|
||||
// don't need to unsubscribe, service does so before sending the quit signal
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return rpcSub, nil
|
||||
}
|
||||
|
||||
// StateDiffAt returns a state diff payload at the specific blockheight
|
||||
func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) {
|
||||
return api.sds.StateDiffAt(blockNumber, params)
|
||||
}
|
||||
|
||||
// StateDiffFor returns a state diff payload for the specific blockhash
|
||||
func (api *PublicStateDiffAPI) StateDiffFor(ctx context.Context, blockHash common.Hash, params Params) (*Payload, error) {
|
||||
return api.sds.StateDiffFor(blockHash, params)
|
||||
}
|
||||
|
||||
// StateTrieAt returns a state trie payload at the specific blockheight
|
||||
func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) {
|
||||
return api.sds.StateTrieAt(blockNumber, params)
|
||||
}
|
||||
|
||||
// StreamCodeAndCodeHash writes all of the codehash=>code pairs out to a websocket channel
|
||||
func (api *PublicStateDiffAPI) StreamCodeAndCodeHash(ctx context.Context, blockNumber uint64) (*rpc.Subscription, error) {
|
||||
// ensure that the RPC connection supports subscriptions
|
||||
notifier, supported := rpc.NotifierFromContext(ctx)
|
||||
if !supported {
|
||||
return nil, rpc.ErrNotificationsUnsupported
|
||||
}
|
||||
|
||||
// create subscription and start waiting for events
|
||||
rpcSub := notifier.CreateSubscription()
|
||||
payloadChan := make(chan types.CodeAndCodeHash, chainEventChanSize)
|
||||
quitChan := make(chan bool)
|
||||
api.sds.StreamCodeAndCodeHash(blockNumber, payloadChan, quitChan)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case payload := <-payloadChan:
|
||||
if err := notifier.Notify(rpcSub.ID, payload); err != nil {
|
||||
log.Error("Failed to send code and codehash packet", "err", err)
|
||||
return
|
||||
}
|
||||
case err := <-rpcSub.Err():
|
||||
log.Error("State diff service rpcSub error", "err", err)
|
||||
return
|
||||
case <-quitChan:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return rpcSub, nil
|
||||
}
|
||||
|
||||
// WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight
|
||||
func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params Params) JobID {
|
||||
var err error
|
||||
start, logger := countApiRequestBegin("writeStateDiffAt", blockNumber)
|
||||
defer countApiRequestEnd(start, logger, err)
|
||||
|
||||
return api.sds.WriteStateDiffAt(blockNumber, params)
|
||||
}
|
||||
|
||||
// WriteStateDiffFor writes a state diff object directly to DB for the specific block hash
|
||||
func (api *PublicStateDiffAPI) WriteStateDiffFor(ctx context.Context, blockHash common.Hash, params Params) error {
|
||||
var err error
|
||||
start, logger := countApiRequestBegin("writeStateDiffFor", blockHash.Hex())
|
||||
defer countApiRequestEnd(start, logger, err)
|
||||
|
||||
err = api.sds.WriteStateDiffFor(blockHash, params)
|
||||
return err
|
||||
}
|
||||
|
||||
// WatchAddress changes the list of watched addresses to which the direct indexing is restricted according to given operation
|
||||
func (api *PublicStateDiffAPI) WatchAddress(operation types.OperationType, args []types.WatchAddressArg) error {
|
||||
return api.sds.WatchAddress(operation, args)
|
||||
}
|
||||
|
||||
// StreamWrites sets up a subscription that streams the status of completed calls to WriteStateDiff*
|
||||
func (api *PublicStateDiffAPI) StreamWrites(ctx context.Context) (*rpc.Subscription, error) {
|
||||
// ensure that the RPC connection supports subscriptions
|
||||
notifier, supported := rpc.NotifierFromContext(ctx)
|
||||
if !supported {
|
||||
return nil, rpc.ErrNotificationsUnsupported
|
||||
}
|
||||
|
||||
// create subscription and start waiting for events
|
||||
rpcSub := notifier.CreateSubscription()
|
||||
|
||||
go func() {
|
||||
// subscribe to events from the statediff service
|
||||
statusChan := make(chan JobStatus, chainEventChanSize)
|
||||
quitChan := make(chan bool, 1)
|
||||
api.sds.SubscribeWriteStatus(rpcSub.ID, statusChan, quitChan)
|
||||
|
||||
var err error
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if err = api.sds.UnsubscribeWriteStatus(rpcSub.ID); err != nil {
|
||||
log.Error("Failed to unsubscribe from job status stream: " + err.Error())
|
||||
}
|
||||
}
|
||||
}()
|
||||
// loop and await payloads and relay them to the subscriber with the notifier
|
||||
for {
|
||||
select {
|
||||
case status := <-statusChan:
|
||||
if err = notifier.Notify(rpcSub.ID, status); err != nil {
|
||||
log.Error("Failed to send job status; error: " + err.Error())
|
||||
return
|
||||
}
|
||||
case err = <-rpcSub.Err():
|
||||
if err != nil {
|
||||
log.Error("State diff service rpcSub error: " + err.Error())
|
||||
return
|
||||
}
|
||||
case <-quitChan:
|
||||
// don't need to unsubscribe, service does so before sending the quit signal
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return rpcSub, nil
|
||||
}
|
||||
934
statediff/builder.go
Normal file
934
statediff/builder.go
Normal file
@ -0,0 +1,934 @@
|
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Contains a batch of utility type declarations used by the tests. As the node
|
||||
// operates on unique types, a lot of them are needed to check various features.
|
||||
|
||||
package statediff
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
metrics2 "github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/state"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/statediff/trie_helpers"
|
||||
types2 "github.com/ethereum/go-ethereum/statediff/types"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
)
|
||||
|
||||
var (
|
||||
nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
|
||||
emptyNode, _ = rlp.EncodeToBytes(&[]byte{})
|
||||
emptyContractRoot = crypto.Keccak256Hash(emptyNode)
|
||||
nullCodeHash = crypto.Keccak256Hash([]byte{}).Bytes()
|
||||
)
|
||||
|
||||
// Builder interface exposes the method for building a state diff between two blocks
|
||||
type Builder interface {
|
||||
BuildStateDiffObject(args Args, params Params) (types2.StateObject, error)
|
||||
BuildStateTrieObject(current *types.Block) (types2.StateObject, error)
|
||||
WriteStateDiffObject(args Args, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error
|
||||
}
|
||||
|
||||
type StateDiffBuilder struct {
|
||||
StateCache state.Database
|
||||
}
|
||||
|
||||
type IterPair struct {
|
||||
Older, Newer trie.NodeIterator
|
||||
}
|
||||
|
||||
// convenience
|
||||
func StateNodeAppender(nodes *[]types2.StateNode) types2.StateNodeSink {
|
||||
return func(node types2.StateNode) error {
|
||||
*nodes = append(*nodes, node)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
func StorageNodeAppender(nodes *[]types2.StorageNode) types2.StorageNodeSink {
|
||||
return func(node types2.StorageNode) error {
|
||||
*nodes = append(*nodes, node)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
func CodeMappingAppender(codeAndCodeHashes *[]types2.CodeAndCodeHash) types2.CodeSink {
|
||||
return func(c types2.CodeAndCodeHash) error {
|
||||
*codeAndCodeHashes = append(*codeAndCodeHashes, c)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewBuilder is used to create a statediff builder
|
||||
func NewBuilder(stateCache state.Database) Builder {
|
||||
return &StateDiffBuilder{
|
||||
StateCache: stateCache, // state cache is safe for concurrent reads
|
||||
}
|
||||
}
|
||||
|
||||
// BuildStateTrieObject builds a state trie object from the provided block
|
||||
func (sdb *StateDiffBuilder) BuildStateTrieObject(current *types.Block) (types2.StateObject, error) {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.BuildStateTrieObjectTimer)
|
||||
currentTrie, err := sdb.StateCache.OpenTrie(current.Root())
|
||||
if err != nil {
|
||||
return types2.StateObject{}, fmt.Errorf("error creating trie for block %d: %v", current.Number(), err)
|
||||
}
|
||||
it := currentTrie.NodeIterator([]byte{})
|
||||
stateNodes, codeAndCodeHashes, err := sdb.buildStateTrie(it)
|
||||
if err != nil {
|
||||
return types2.StateObject{}, fmt.Errorf("error collecting state nodes for block %d: %v", current.Number(), err)
|
||||
}
|
||||
return types2.StateObject{
|
||||
BlockNumber: current.Number(),
|
||||
BlockHash: current.Hash(),
|
||||
Nodes: stateNodes,
|
||||
CodeAndCodeHashes: codeAndCodeHashes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (sdb *StateDiffBuilder) buildStateTrie(it trie.NodeIterator) ([]types2.StateNode, []types2.CodeAndCodeHash, error) {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.BuildStateTrieTimer)
|
||||
stateNodes := make([]types2.StateNode, 0)
|
||||
codeAndCodeHashes := make([]types2.CodeAndCodeHash, 0)
|
||||
for it.Next(true) {
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case types2.Leaf:
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
|
||||
return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
|
||||
}
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
node.LeafKey = leafKey
|
||||
if !bytes.Equal(account.CodeHash, nullCodeHash) {
|
||||
var storageNodes []types2.StorageNode
|
||||
err := sdb.buildStorageNodesEventual(account.Root, true, StorageNodeAppender(&storageNodes))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err)
|
||||
}
|
||||
node.StorageNodes = storageNodes
|
||||
// emit codehash => code mappings for cod
|
||||
codeHash := common.BytesToHash(account.CodeHash)
|
||||
code, err := sdb.StateCache.ContractCode(common.Hash{}, codeHash)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
|
||||
}
|
||||
codeAndCodeHashes = append(codeAndCodeHashes, types2.CodeAndCodeHash{
|
||||
Hash: codeHash,
|
||||
Code: code,
|
||||
})
|
||||
}
|
||||
stateNodes = append(stateNodes, node)
|
||||
case types2.Extension, types2.Branch:
|
||||
stateNodes = append(stateNodes, node)
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return stateNodes, codeAndCodeHashes, it.Error()
|
||||
}
|
||||
|
||||
// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
|
||||
func (sdb *StateDiffBuilder) BuildStateDiffObject(args Args, params Params) (types2.StateObject, error) {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.BuildStateDiffObjectTimer)
|
||||
var stateNodes []types2.StateNode
|
||||
var codeAndCodeHashes []types2.CodeAndCodeHash
|
||||
err := sdb.WriteStateDiffObject(args, params, StateNodeAppender(&stateNodes), CodeMappingAppender(&codeAndCodeHashes))
|
||||
if err != nil {
|
||||
return types2.StateObject{}, err
|
||||
}
|
||||
return types2.StateObject{
|
||||
BlockHash: args.BlockHash,
|
||||
BlockNumber: args.BlockNumber,
|
||||
Nodes: stateNodes,
|
||||
CodeAndCodeHashes: codeAndCodeHashes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// WriteStateDiffObject writes a statediff object to output callback
|
||||
func (sdb *StateDiffBuilder) WriteStateDiffObject(args Args, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.WriteStateDiffObjectTimer)
|
||||
// Load tries for old and new states
|
||||
oldTrie, err := sdb.StateCache.OpenTrie(args.OldStateRoot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
|
||||
}
|
||||
newTrie, err := sdb.StateCache.OpenTrie(args.NewStateRoot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating trie for newStateRoot: %v", err)
|
||||
}
|
||||
|
||||
// we do two state trie iterations:
|
||||
// one for new/updated nodes,
|
||||
// one for deleted/updated nodes;
|
||||
// prepare 2 iterator instances for each task
|
||||
iterPairs := []IterPair{
|
||||
{
|
||||
Older: oldTrie.NodeIterator([]byte{}),
|
||||
Newer: newTrie.NodeIterator([]byte{}),
|
||||
},
|
||||
{
|
||||
Older: oldTrie.NodeIterator([]byte{}),
|
||||
Newer: newTrie.NodeIterator([]byte{}),
|
||||
},
|
||||
}
|
||||
|
||||
logger := log.New("hash", args.BlockHash.Hex(), "number", args.BlockNumber)
|
||||
if !params.IntermediateStateNodes {
|
||||
return sdb.BuildStateDiffWithoutIntermediateStateNodes(iterPairs, params, output, codeOutput, logger)
|
||||
} else {
|
||||
return sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs, params, output, codeOutput, logger)
|
||||
}
|
||||
}
|
||||
|
||||
func (sdb *StateDiffBuilder) BuildStateDiffWithIntermediateStateNodes(iterPairs []IterPair, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink, logger log.Logger) error {
|
||||
logger.Debug("statediff BEGIN BuildStateDiffWithIntermediateStateNodes")
|
||||
defer metrics2.ReportAndUpdateDuration("statediff END BuildStateDiffWithIntermediateStateNodes", time.Now(), logger, metrics2.IndexerMetrics.BuildStateDiffWithIntermediateStateNodesTimer)
|
||||
// collect a slice of all the nodes that were touched and exist at B (B-A)
|
||||
// a map of their leafkey to all the accounts that were touched and exist at B
|
||||
// and a slice of all the paths for the nodes in both of the above sets
|
||||
diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStateWithIntermediateNodes(
|
||||
iterPairs[0].Older, iterPairs[0].Newer, params.watchedAddressesLeafPaths, output, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
|
||||
}
|
||||
|
||||
// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
|
||||
// a map of their leafkey to all the accounts that were touched and exist at A
|
||||
diffAccountsAtA, err := sdb.deletedOrUpdatedState(
|
||||
iterPairs[1].Older, iterPairs[1].Newer,
|
||||
diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafPaths,
|
||||
params.IntermediateStateNodes, params.IntermediateStorageNodes, output, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
|
||||
}
|
||||
|
||||
// collect and sort the leafkey keys for both account mappings into a slice
|
||||
t := time.Now()
|
||||
createKeys := trie_helpers.SortKeys(diffAccountsAtB)
|
||||
deleteKeys := trie_helpers.SortKeys(diffAccountsAtA)
|
||||
logger.Debug(fmt.Sprintf("statediff BuildStateDiffWithIntermediateStateNodes sort duration=%dms", time.Since(t).Milliseconds()))
|
||||
|
||||
// and then find the intersection of these keys
|
||||
// these are the leafkeys for the accounts which exist at both A and B but are different
|
||||
// this also mutates the passed in createKeys and deleteKeys, removing the intersection keys
|
||||
// and leaving the truly created or deleted keys in place
|
||||
t = time.Now()
|
||||
updatedKeys := trie_helpers.FindIntersection(createKeys, deleteKeys)
|
||||
logger.Debug(fmt.Sprintf("statediff BuildStateDiffWithIntermediateStateNodes intersection count=%d duration=%dms",
|
||||
len(updatedKeys),
|
||||
time.Since(t).Milliseconds()))
|
||||
|
||||
// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
|
||||
err = sdb.buildAccountUpdates(
|
||||
diffAccountsAtB, diffAccountsAtA, updatedKeys,
|
||||
params.IntermediateStorageNodes, output, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error building diff for updated accounts: %v", err)
|
||||
}
|
||||
// build the diff nodes for created accounts
|
||||
err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error building diff for created accounts: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdb *StateDiffBuilder) BuildStateDiffWithoutIntermediateStateNodes(iterPairs []IterPair, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink, logger log.Logger) error {
|
||||
logger.Debug("statediff BEGIN BuildStateDiffWithoutIntermediateStateNodes")
|
||||
defer metrics2.ReportAndUpdateDuration("statediff END BuildStateDiffWithoutIntermediateStateNodes", time.Now(), logger, metrics2.IndexerMetrics.BuildStateDiffWithoutIntermediateStateNodesTimer)
|
||||
// collect a map of their leafkey to all the accounts that were touched and exist at B
|
||||
// and a slice of all the paths for the nodes in both of the above sets
|
||||
diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState(
|
||||
iterPairs[0].Older, iterPairs[0].Newer,
|
||||
params.watchedAddressesLeafPaths, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
|
||||
}
|
||||
|
||||
// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
|
||||
// a map of their leafkey to all the accounts that were touched and exist at A
|
||||
diffAccountsAtA, err := sdb.deletedOrUpdatedState(
|
||||
iterPairs[1].Older, iterPairs[1].Newer,
|
||||
diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafPaths,
|
||||
params.IntermediateStateNodes, params.IntermediateStorageNodes, output, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
|
||||
}
|
||||
|
||||
// collect and sort the leafkeys for both account mappings into a slice
|
||||
t := time.Now()
|
||||
createKeys := trie_helpers.SortKeys(diffAccountsAtB)
|
||||
deleteKeys := trie_helpers.SortKeys(diffAccountsAtA)
|
||||
logger.Debug(fmt.Sprintf("statediff BuildStateDiffWithoutIntermediateStateNodessort sort duration=%dms", time.Since(t).Milliseconds()))
|
||||
|
||||
// and then find the intersection of these keys
|
||||
// these are the leafkeys for the accounts which exist at both A and B but are different
|
||||
// this also mutates the passed in createKeys and deleteKeys, removing in intersection keys
|
||||
// and leaving the truly created or deleted keys in place
|
||||
t = time.Now()
|
||||
updatedKeys := trie_helpers.FindIntersection(createKeys, deleteKeys)
|
||||
logger.Debug(fmt.Sprintf("statediff BuildStateDiffWithoutIntermediateStateNodes intersection count=%d duration=%dms",
|
||||
len(updatedKeys),
|
||||
time.Since(t).Milliseconds()))
|
||||
|
||||
// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
|
||||
err = sdb.buildAccountUpdates(
|
||||
diffAccountsAtB, diffAccountsAtA, updatedKeys,
|
||||
params.IntermediateStorageNodes, output, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error building diff for updated accounts: %v", err)
|
||||
}
|
||||
// build the diff nodes for created accounts
|
||||
err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error building diff for created accounts: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createdAndUpdatedState returns
|
||||
// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
|
||||
// and a slice of the paths for all of the nodes included in both
|
||||
func (sdb *StateDiffBuilder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddressesLeafPaths [][]byte, logger log.Logger) (types2.AccountMap, map[string]bool, error) {
|
||||
logger.Debug("statediff BEGIN createdAndUpdatedState")
|
||||
defer metrics2.ReportAndUpdateDuration("statediff END createdAndUpdatedState", time.Now(), logger, metrics2.IndexerMetrics.CreatedAndUpdatedStateTimer)
|
||||
diffPathsAtB := make(map[string]bool)
|
||||
diffAccountsAtB := make(types2.AccountMap)
|
||||
watchingAddresses := len(watchedAddressesLeafPaths) > 0
|
||||
|
||||
it, _ := trie.NewDifferenceIterator(a, b)
|
||||
for it.Next(true) {
|
||||
// ignore node if it is not along paths of interest
|
||||
if watchingAddresses && !isValidPrefixPath(watchedAddressesLeafPaths, it.Path()) {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
|
||||
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if node.NodeType == types2.Leaf {
|
||||
// created vs updated is important for leaf nodes since we need to diff their storage
|
||||
// so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
|
||||
return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
|
||||
}
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
|
||||
// ignore leaf node if it is not a watched address
|
||||
if !isWatchedAddress(watchedAddressesLeafPaths, valueNodePath) {
|
||||
continue
|
||||
}
|
||||
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
diffAccountsAtB[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
Account: &account,
|
||||
}
|
||||
}
|
||||
// add both intermediate and leaf node paths to the list of diffPathsAtB
|
||||
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
|
||||
}
|
||||
return diffAccountsAtB, diffPathsAtB, it.Error()
|
||||
}
|
||||
|
||||
// createdAndUpdatedStateWithIntermediateNodes returns
|
||||
// a slice of all the intermediate nodes that exist in a different state at B than A
|
||||
// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
|
||||
// and a slice of the paths for all of the nodes included in both
|
||||
func (sdb *StateDiffBuilder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator, watchedAddressesLeafPaths [][]byte, output types2.StateNodeSink, logger log.Logger) (types2.AccountMap, map[string]bool, error) {
|
||||
logger.Debug("statediff BEGIN createdAndUpdatedStateWithIntermediateNodes")
|
||||
defer metrics2.ReportAndUpdateDuration("statediff END createdAndUpdatedStateWithIntermediateNodes", time.Now(), logger, metrics2.IndexerMetrics.CreatedAndUpdatedStateWithIntermediateNodesTimer)
|
||||
diffPathsAtB := make(map[string]bool)
|
||||
diffAccountsAtB := make(types2.AccountMap)
|
||||
watchingAddresses := len(watchedAddressesLeafPaths) > 0
|
||||
|
||||
it, itCount := trie.NewDifferenceIterator(a, b)
|
||||
for it.Next(true) {
|
||||
// ignore node if it is not along paths of interest
|
||||
if watchingAddresses && !isValidPrefixPath(watchedAddressesLeafPaths, it.Path()) {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case types2.Leaf:
|
||||
// created vs updated is important for leaf nodes since we need to diff their storage
|
||||
// so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
|
||||
return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
|
||||
}
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
|
||||
// ignore leaf node if it is not a watched address
|
||||
if !isWatchedAddress(watchedAddressesLeafPaths, valueNodePath) {
|
||||
continue
|
||||
}
|
||||
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
diffAccountsAtB[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
Account: &account,
|
||||
}
|
||||
case types2.Extension, types2.Branch:
|
||||
// create a diff for any intermediate node that has changed at b
|
||||
// created vs updated makes no difference for intermediate nodes since we do not need to diff storage
|
||||
if err := output(types2.StateNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
// add both intermediate and leaf node paths to the list of diffPathsAtB
|
||||
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
|
||||
}
|
||||
logger.Debug("statediff COUNTS createdAndUpdatedStateWithIntermediateNodes", "it", itCount, "diffAccountsAtB", len(diffAccountsAtB), "diffPathsAtB", len(diffPathsAtB))
|
||||
metrics2.IndexerMetrics.DifferenceIteratorCounter.Inc(int64(*itCount))
|
||||
return diffAccountsAtB, diffPathsAtB, it.Error()
|
||||
}
|
||||
|
||||
// deletedOrUpdatedState returns a slice of all the pathes that are emptied at B
|
||||
// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B
|
||||
func (sdb *StateDiffBuilder) deletedOrUpdatedState(a, b trie.NodeIterator, diffAccountsAtB types2.AccountMap, diffPathsAtB map[string]bool, watchedAddressesLeafPaths [][]byte, intermediateStateNodes, intermediateStorageNodes bool, output types2.StateNodeSink, logger log.Logger) (types2.AccountMap, error) {
|
||||
logger.Debug("statediff BEGIN deletedOrUpdatedState")
|
||||
defer metrics2.ReportAndUpdateDuration("statediff END deletedOrUpdatedState", time.Now(), logger, metrics2.IndexerMetrics.DeletedOrUpdatedStateTimer)
|
||||
diffAccountAtA := make(types2.AccountMap)
|
||||
watchingAddresses := len(watchedAddressesLeafPaths) > 0
|
||||
|
||||
it, _ := trie.NewDifferenceIterator(b, a)
|
||||
for it.Next(true) {
|
||||
// ignore node if it is not along paths of interest
|
||||
if watchingAddresses && !isValidPrefixPath(watchedAddressesLeafPaths, it.Path()) {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
|
||||
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case types2.Leaf:
|
||||
// map all different accounts at A to their leafkey
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
|
||||
return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
|
||||
}
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
|
||||
// ignore leaf node if it is not a watched address
|
||||
if !isWatchedAddress(watchedAddressesLeafPaths, valueNodePath) {
|
||||
continue
|
||||
}
|
||||
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
diffAccountAtA[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
Account: &account,
|
||||
}
|
||||
// if this node's path did not show up in diffPathsAtB
|
||||
// that means the node at this path was deleted (or moved) in B
|
||||
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
|
||||
var diff types2.StateNode
|
||||
// if this node's leaf key also did not show up in diffAccountsAtB
|
||||
// that means the node was deleted
|
||||
// in that case, emit an empty "removed" diff state node
|
||||
// include empty "removed" diff storage nodes for all the storage slots
|
||||
if _, ok := diffAccountsAtB[common.Bytes2Hex(leafKey)]; !ok {
|
||||
diff = types2.StateNode{
|
||||
NodeType: types2.Removed,
|
||||
Path: node.Path,
|
||||
LeafKey: leafKey,
|
||||
NodeValue: []byte{},
|
||||
}
|
||||
|
||||
var storageDiffs []types2.StorageNode
|
||||
err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, StorageNodeAppender(&storageDiffs))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err)
|
||||
}
|
||||
diff.StorageNodes = storageDiffs
|
||||
} else {
|
||||
// emit an empty "removed" diff with empty leaf key if the account was moved
|
||||
diff = types2.StateNode{
|
||||
NodeType: types2.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
}
|
||||
}
|
||||
|
||||
if err := output(diff); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
case types2.Extension, types2.Branch:
|
||||
// if this node's path did not show up in diffPathsAtB
|
||||
// that means the node at this path was deleted (or moved) in B
|
||||
// emit an empty "removed" diff to signify as such
|
||||
if intermediateStateNodes {
|
||||
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
|
||||
if err := output(types2.StateNode{
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
NodeType: types2.Removed,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
// fall through, we did everything we need to do with these node types
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return diffAccountAtA, it.Error()
|
||||
}
|
||||
|
||||
// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys
|
||||
// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
|
||||
// needs to be called before building account creations and deletions as this mutates
|
||||
// those account maps to remove the accounts which were updated
|
||||
func (sdb *StateDiffBuilder) buildAccountUpdates(creations, deletions types2.AccountMap, updatedKeys []string, intermediateStorageNodes bool, output types2.StateNodeSink, logger log.Logger) error {
|
||||
logger.Debug("statediff BEGIN buildAccountUpdates", "creations", len(creations), "deletions", len(deletions), "updatedKeys", len(updatedKeys))
|
||||
defer metrics2.ReportAndUpdateDuration("statediff END buildAccountUpdates ", time.Now(), logger, metrics2.IndexerMetrics.BuildAccountUpdatesTimer)
|
||||
var err error
|
||||
for _, key := range updatedKeys {
|
||||
createdAcc := creations[key]
|
||||
deletedAcc := deletions[key]
|
||||
var storageDiffs []types2.StorageNode
|
||||
if deletedAcc.Account != nil && createdAcc.Account != nil {
|
||||
oldSR := deletedAcc.Account.Root
|
||||
newSR := createdAcc.Account.Root
|
||||
err = sdb.buildStorageNodesIncremental(
|
||||
oldSR, newSR, intermediateStorageNodes,
|
||||
StorageNodeAppender(&storageDiffs))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err)
|
||||
}
|
||||
}
|
||||
if err = output(types2.StateNode{
|
||||
NodeType: createdAcc.NodeType,
|
||||
Path: createdAcc.Path,
|
||||
NodeValue: createdAcc.NodeValue,
|
||||
LeafKey: createdAcc.LeafKey,
|
||||
StorageNodes: storageDiffs,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(creations, key)
|
||||
delete(deletions, key)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
|
||||
// it also returns the code and codehash for created contract accounts
|
||||
func (sdb *StateDiffBuilder) buildAccountCreations(accounts types2.AccountMap, intermediateStorageNodes bool, output types2.StateNodeSink, codeOutput types2.CodeSink, logger log.Logger) error {
|
||||
logger.Debug("statediff BEGIN buildAccountCreations")
|
||||
defer metrics2.ReportAndUpdateDuration("statediff END buildAccountCreations", time.Now(), logger, metrics2.IndexerMetrics.BuildAccountCreationsTimer)
|
||||
for _, val := range accounts {
|
||||
diff := types2.StateNode{
|
||||
NodeType: val.NodeType,
|
||||
Path: val.Path,
|
||||
LeafKey: val.LeafKey,
|
||||
NodeValue: val.NodeValue,
|
||||
}
|
||||
if !bytes.Equal(val.Account.CodeHash, nullCodeHash) {
|
||||
// For contract creations, any storage node contained is a diff
|
||||
var storageDiffs []types2.StorageNode
|
||||
err := sdb.buildStorageNodesEventual(val.Account.Root, intermediateStorageNodes, StorageNodeAppender(&storageDiffs))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err)
|
||||
}
|
||||
diff.StorageNodes = storageDiffs
|
||||
// emit codehash => code mappings for cod
|
||||
codeHash := common.BytesToHash(val.Account.CodeHash)
|
||||
code, err := sdb.StateCache.ContractCode(common.Hash{}, codeHash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
|
||||
}
|
||||
if err := codeOutput(types2.CodeAndCodeHash{
|
||||
Hash: codeHash,
|
||||
Code: code,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := output(diff); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildStorageNodesEventual builds the storage diff node objects for a created account
|
||||
// i.e. it returns all the storage nodes at this state, since there is no previous state
|
||||
func (sdb *StateDiffBuilder) buildStorageNodesEventual(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.BuildStorageNodesEventualTimer)
|
||||
if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
|
||||
return nil
|
||||
}
|
||||
log.Debug("Storage Root For Eventual Diff", "root", sr.Hex())
|
||||
sTrie, err := sdb.StateCache.OpenTrie(sr)
|
||||
if err != nil {
|
||||
log.Info("error in build storage diff eventual", "error", err)
|
||||
return err
|
||||
}
|
||||
it := sTrie.NodeIterator(make([]byte, 0))
|
||||
err = sdb.buildStorageNodesFromTrie(it, intermediateNodes, output)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator
|
||||
// including intermediate nodes can be turned on or off
|
||||
func (sdb *StateDiffBuilder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.BuildStorageNodesFromTrieTimer)
|
||||
for it.Next(true) {
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case types2.Leaf:
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
if err := output(types2.StorageNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
case types2.Extension, types2.Branch:
|
||||
if intermediateNodes {
|
||||
if err := output(types2.StorageNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return it.Error()
|
||||
}
|
||||
|
||||
// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account
|
||||
func (sdb *StateDiffBuilder) buildRemovedAccountStorageNodes(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.BuildRemovedAccountStorageNodesTimer)
|
||||
if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
|
||||
return nil
|
||||
}
|
||||
log.Debug("Storage Root For Removed Diffs", "root", sr.Hex())
|
||||
sTrie, err := sdb.StateCache.OpenTrie(sr)
|
||||
if err != nil {
|
||||
log.Info("error in build removed account storage diffs", "error", err)
|
||||
return err
|
||||
}
|
||||
it := sTrie.NodeIterator(make([]byte, 0))
|
||||
err = sdb.buildRemovedStorageNodesFromTrie(it, intermediateNodes, output)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator
|
||||
// including intermediate nodes can be turned on or off
|
||||
func (sdb *StateDiffBuilder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.BuildRemovedStorageNodesFromTrieTimer)
|
||||
for it.Next(true) {
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case types2.Leaf:
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
if err := output(types2.StorageNode{
|
||||
NodeType: types2.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
LeafKey: leafKey,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
case types2.Extension, types2.Branch:
|
||||
if intermediateNodes {
|
||||
if err := output(types2.StorageNode{
|
||||
NodeType: types2.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return it.Error()
|
||||
}
|
||||
|
||||
// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
|
||||
func (sdb *StateDiffBuilder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.BuildStorageNodesIncrementalTimer)
|
||||
if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) {
|
||||
return nil
|
||||
}
|
||||
log.Trace("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
|
||||
oldTrie, err := sdb.StateCache.OpenTrie(oldSR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newTrie, err := sdb.StateCache.OpenTrie(newSR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
diffSlotsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStorage(
|
||||
oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
|
||||
intermediateNodes, output)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
|
||||
diffSlotsAtB, diffPathsAtB, intermediateNodes, output)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdb *StateDiffBuilder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) (map[string]bool, map[string]bool, error) {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.CreatedAndUpdatedStorageTimer)
|
||||
diffPathsAtB := make(map[string]bool)
|
||||
diffSlotsAtB := make(map[string]bool)
|
||||
it, _ := trie.NewDifferenceIterator(a, b)
|
||||
for it.Next(true) {
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
switch node.NodeType {
|
||||
case types2.Leaf:
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
diffSlotsAtB[common.Bytes2Hex(leafKey)] = true
|
||||
if err := output(types2.StorageNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
LeafKey: leafKey,
|
||||
}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
case types2.Extension, types2.Branch:
|
||||
if intermediateNodes {
|
||||
if err := output(types2.StorageNode{
|
||||
NodeType: node.NodeType,
|
||||
Path: node.Path,
|
||||
NodeValue: node.NodeValue,
|
||||
}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
|
||||
}
|
||||
return diffSlotsAtB, diffPathsAtB, it.Error()
|
||||
}
|
||||
|
||||
func (sdb *StateDiffBuilder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffSlotsAtB, diffPathsAtB map[string]bool, intermediateNodes bool, output types2.StorageNodeSink) error {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.DeletedOrUpdatedStorageTimer)
|
||||
it, _ := trie.NewDifferenceIterator(b, a)
|
||||
for it.Next(true) {
|
||||
// skip value nodes
|
||||
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
|
||||
continue
|
||||
}
|
||||
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch node.NodeType {
|
||||
case types2.Leaf:
|
||||
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
|
||||
valueNodePath := append(node.Path, partialPath...)
|
||||
encodedPath := trie.HexToCompact(valueNodePath)
|
||||
leafKey := encodedPath[1:]
|
||||
|
||||
// if this node's path did not show up in diffPathsAtB
|
||||
// that means the node at this path was deleted (or moved) in B
|
||||
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
|
||||
// if this node's leaf key also did not show up in diffSlotsAtB
|
||||
// that means the node was deleted
|
||||
// in that case, emit an empty "removed" diff storage node
|
||||
if _, ok := diffSlotsAtB[common.Bytes2Hex(leafKey)]; !ok {
|
||||
if err := output(types2.StorageNode{
|
||||
NodeType: types2.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
LeafKey: leafKey,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// emit an empty "removed" diff with empty leaf key if the account was moved
|
||||
if err := output(types2.StorageNode{
|
||||
NodeType: types2.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
case types2.Extension, types2.Branch:
|
||||
// if this node's path did not show up in diffPathsAtB
|
||||
// that means the node at this path was deleted in B
|
||||
// in that case, emit an empty "removed" diff storage node
|
||||
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
|
||||
if intermediateNodes {
|
||||
if err := output(types2.StorageNode{
|
||||
NodeType: types2.Removed,
|
||||
Path: node.Path,
|
||||
NodeValue: []byte{},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unexpected node type %s", node.NodeType)
|
||||
}
|
||||
}
|
||||
return it.Error()
|
||||
}
|
||||
|
||||
// isValidPrefixPath is used to check if a node at currentPath is a parent | ancestor to one of the addresses the builder is configured to watch
|
||||
func isValidPrefixPath(watchedAddressesLeafPaths [][]byte, currentPath []byte) bool {
|
||||
for _, watchedAddressPath := range watchedAddressesLeafPaths {
|
||||
if bytes.HasPrefix(watchedAddressPath, currentPath) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch
|
||||
func isWatchedAddress(watchedAddressesLeafPaths [][]byte, valueNodePath []byte) bool {
|
||||
defer metrics2.UpdateDuration(time.Now(), metrics2.IndexerMetrics.IsWatchedAddressTimer)
|
||||
// If we aren't watching any specific addresses, we are watching everything
|
||||
if len(watchedAddressesLeafPaths) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, watchedAddressPath := range watchedAddressesLeafPaths {
|
||||
if bytes.Equal(watchedAddressPath, valueNodePath) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
2600
statediff/builder_test.go
Normal file
2600
statediff/builder_test.go
Normal file
File diff suppressed because it is too large
Load Diff
89
statediff/config.go
Normal file
89
statediff/config.go
Normal file
@ -0,0 +1,89 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package statediff
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/big"
|
||||
"sync"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
)
|
||||
|
||||
// Config contains instantiation parameters for the state diffing service
|
||||
type Config struct {
|
||||
// The configuration used for the stateDiff Indexer
|
||||
IndexerConfig interfaces.Config
|
||||
// A unique ID used for this service
|
||||
ID string
|
||||
// Name for the client this service is running
|
||||
ClientName string
|
||||
// Whether to enable writing state diffs directly to track blockchain head
|
||||
EnableWriteLoop bool
|
||||
// Size of the worker pool
|
||||
NumWorkers uint
|
||||
// Should the statediff service wait until geth has synced to the head of the blockchain?
|
||||
WaitForSync bool
|
||||
// Context
|
||||
Context context.Context
|
||||
}
|
||||
|
||||
// Params contains config parameters for the state diff builder
|
||||
type Params struct {
|
||||
IntermediateStateNodes bool
|
||||
IntermediateStorageNodes bool
|
||||
IncludeBlock bool
|
||||
IncludeReceipts bool
|
||||
IncludeTD bool
|
||||
IncludeCode bool
|
||||
WatchedAddresses []common.Address
|
||||
watchedAddressesLeafPaths [][]byte
|
||||
}
|
||||
|
||||
// ComputeWatchedAddressesLeafPaths populates a slice with paths (hex_encoding(Keccak256)) of each of the WatchedAddresses
|
||||
func (p *Params) ComputeWatchedAddressesLeafPaths() {
|
||||
p.watchedAddressesLeafPaths = make([][]byte, len(p.WatchedAddresses))
|
||||
for i, address := range p.WatchedAddresses {
|
||||
p.watchedAddressesLeafPaths[i] = keybytesToHex(crypto.Keccak256(address.Bytes()))
|
||||
}
|
||||
}
|
||||
|
||||
// ParamsWithMutex allows to lock the parameters while they are being updated | read from
|
||||
type ParamsWithMutex struct {
|
||||
Params
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
// Args bundles the arguments for the state diff builder
|
||||
type Args struct {
|
||||
OldStateRoot, NewStateRoot, BlockHash common.Hash
|
||||
BlockNumber *big.Int
|
||||
}
|
||||
|
||||
// https://github.com/ethereum/go-ethereum/blob/master/trie/encoding.go#L97
|
||||
func keybytesToHex(str []byte) []byte {
|
||||
l := len(str)*2 + 1
|
||||
var nibbles = make([]byte, l)
|
||||
for i, b := range str {
|
||||
nibbles[i*2] = b / 16
|
||||
nibbles[i*2+1] = b % 16
|
||||
}
|
||||
nibbles[l-1] = 16
|
||||
return nibbles
|
||||
}
|
||||
17
statediff/docs/KnownGaps.md
Normal file
17
statediff/docs/KnownGaps.md
Normal file
@ -0,0 +1,17 @@
|
||||
# Overview
|
||||
|
||||
This document will provide some insight into the `known_gaps` table, their use cases, and implementation. Please refer to the [following PR](https://github.com/vulcanize/go-ethereum/pull/217) and the [following epic](https://github.com/vulcanize/ops/issues/143) to grasp their inception.
|
||||
|
||||
|
||||
|
||||
# Use Cases
|
||||
|
||||
The known gaps table is updated when the following events occur:
|
||||
|
||||
1. At start up we check the latest block from the `eth.headers_cid` table. We compare the first block that we are processing with the latest block from the DB. If they are not one unit of expectedDifference away from each other, add the gap between the two blocks.
|
||||
2. If there is any error in processing a block (db connection, deadlock, etc), add that block to the knownErrorBlocks slice, when the next block is successfully written, write this slice into the DB.
|
||||
|
||||
# Glossary
|
||||
|
||||
1. `expectedDifference (number)` - This number indicates what the difference between two blocks should be. If we are capturing all events on a geth node then this number would be `1`. But once we scale nodes, the `expectedDifference` might be `2` or greater.
|
||||
2. `processingKey (number)` - This number can be used to keep track of different geth nodes and their specific `expectedDifference`.
|
||||
3
statediff/docs/README.md
Normal file
3
statediff/docs/README.md
Normal file
@ -0,0 +1,3 @@
|
||||
# Overview
|
||||
|
||||
This folder keeps tracks of random documents as they relate to the `statediff` service.
|
||||
21
statediff/docs/database.md
Normal file
21
statediff/docs/database.md
Normal file
@ -0,0 +1,21 @@
|
||||
# Overview
|
||||
|
||||
This document will go through some notes on the database component of the statediff service.
|
||||
|
||||
# Components
|
||||
|
||||
- Indexer: The indexer creates IPLD and DB models to insert to the Postgres DB. It performs the insert utilizing and atomic function.
|
||||
- Builder: The builder constructs the statediff object that needs to be inserted.
|
||||
- Known Gaps: Captures any gaps that might have occured and either writes them to the DB, local sql file, to prometeus, or a local error.
|
||||
|
||||
# Making Code Changes
|
||||
|
||||
## Adding a New Function to the Indexer
|
||||
|
||||
If you want to implement a new feature for adding data to the database. Keep the following in mind:
|
||||
|
||||
1. You need to handle `sql`, `file`, and `dump`.
|
||||
1. `sql` - Contains the code needed to write directly to the `sql` db.
|
||||
2. `file` - Contains all the code required to write the SQL statements to a file.
|
||||
3. `dump` - Contains all the code for outputting events to the console.
|
||||
2. You will have to add it to the `interfaces.StateDiffIndexer` interface.
|
||||
BIN
statediff/docs/diagrams/KnownGapsProcess.png
Normal file
BIN
statediff/docs/diagrams/KnownGapsProcess.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 33 KiB |
63
statediff/helpers.go
Normal file
63
statediff/helpers.go
Normal file
@ -0,0 +1,63 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2022 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package statediff
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
)
|
||||
|
||||
// LoadConfig loads chain config from json file
|
||||
func LoadConfig(chainConfigPath string) (*params.ChainConfig, error) {
|
||||
file, err := os.Open(chainConfigPath)
|
||||
if err != nil {
|
||||
log.Error(fmt.Sprintf("Failed to read chain config file: %v", err))
|
||||
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
chainConfig := new(params.ChainConfig)
|
||||
if err := json.NewDecoder(file).Decode(chainConfig); err != nil {
|
||||
log.Error(fmt.Sprintf("invalid chain config file: %v", err))
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Info(fmt.Sprintf("Using chain config from %s file. Content %+v", chainConfigPath, chainConfig))
|
||||
|
||||
return chainConfig, nil
|
||||
}
|
||||
|
||||
// ChainConfig returns the appropriate ethereum chain config for the provided chain id
|
||||
func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
|
||||
switch chainID {
|
||||
case 1:
|
||||
return params.MainnetChainConfig, nil
|
||||
case 4:
|
||||
return params.RinkebyChainConfig, nil
|
||||
case 5:
|
||||
return params.GoerliChainConfig, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
|
||||
}
|
||||
}
|
||||
81
statediff/indexer/constructor.go
Normal file
81
statediff/indexer/constructor.go
Normal file
@ -0,0 +1,81 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
)
|
||||
|
||||
// NewStateDiffIndexer creates and returns an implementation of the StateDiffIndexer interface.
|
||||
func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, nodeInfo node.Info, config interfaces.Config) (sql.Database, interfaces.StateDiffIndexer, error) {
|
||||
switch config.Type() {
|
||||
case shared.FILE:
|
||||
log.Info("Starting statediff service in SQL file writing mode")
|
||||
fc, ok := config.(file.Config)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("file config is not the correct type: got %T, expected %T", config, file.Config{})
|
||||
}
|
||||
fc.NodeInfo = nodeInfo
|
||||
ind, err := file.NewStateDiffIndexer(ctx, chainConfig, fc)
|
||||
return nil, ind, err
|
||||
case shared.POSTGRES:
|
||||
log.Info("Starting statediff service in Postgres writing mode")
|
||||
pgc, ok := config.(postgres.Config)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("postgres config is not the correct type: got %T, expected %T", config, postgres.Config{})
|
||||
}
|
||||
var err error
|
||||
var driver sql.Driver
|
||||
switch pgc.Driver {
|
||||
case postgres.PGX:
|
||||
driver, err = postgres.NewPGXDriver(ctx, pgc, nodeInfo)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
case postgres.SQLX:
|
||||
driver, err = postgres.NewSQLXDriver(ctx, pgc, nodeInfo)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unrecognized Postgres driver type: %s", pgc.Driver)
|
||||
}
|
||||
db := postgres.NewPostgresDB(driver, pgc.Upsert)
|
||||
ind, err := sql.NewStateDiffIndexer(ctx, chainConfig, db)
|
||||
return db, ind, err
|
||||
case shared.DUMP:
|
||||
log.Info("Starting statediff service in data dump mode")
|
||||
dumpc, ok := config.(dump.Config)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("dump config is not the correct type: got %T, expected %T", config, dump.Config{})
|
||||
}
|
||||
return nil, dump.NewStateDiffIndexer(chainConfig, dumpc), nil
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unrecognized database type: %s", config.Type())
|
||||
}
|
||||
}
|
||||
97
statediff/indexer/database/dump/batch_tx.go
Normal file
97
statediff/indexer/database/dump/batch_tx.go
Normal file
@ -0,0 +1,97 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package dump
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/models"
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
dshelp "github.com/ipfs/go-ipfs-ds-help"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
)
|
||||
|
||||
// BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration
|
||||
type BatchTx struct {
|
||||
BlockNumber string
|
||||
dump io.Writer
|
||||
quit chan struct{}
|
||||
iplds chan models.IPLDModel
|
||||
ipldCache models.IPLDBatch
|
||||
|
||||
submit func(blockTx *BatchTx, err error) error
|
||||
}
|
||||
|
||||
// Submit satisfies indexer.AtomicTx
|
||||
func (tx *BatchTx) Submit(err error) error {
|
||||
return tx.submit(tx, err)
|
||||
}
|
||||
|
||||
func (tx *BatchTx) flush() error {
|
||||
if _, err := fmt.Fprintf(tx.dump, "%+v\r\n", tx.ipldCache); err != nil {
|
||||
return err
|
||||
}
|
||||
tx.ipldCache = models.IPLDBatch{}
|
||||
return nil
|
||||
}
|
||||
|
||||
// run in background goroutine to synchronize concurrent appends to the ipldCache
|
||||
func (tx *BatchTx) cache() {
|
||||
for {
|
||||
select {
|
||||
case i := <-tx.iplds:
|
||||
tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
|
||||
tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
|
||||
case <-tx.quit:
|
||||
tx.ipldCache = models.IPLDBatch{}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (tx *BatchTx) cacheDirect(key string, value []byte) {
|
||||
tx.iplds <- models.IPLDModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
Key: key,
|
||||
Data: value,
|
||||
}
|
||||
}
|
||||
|
||||
func (tx *BatchTx) cacheIPLD(i node.Node) {
|
||||
tx.iplds <- models.IPLDModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(),
|
||||
Data: i.RawData(),
|
||||
}
|
||||
}
|
||||
|
||||
func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error) {
|
||||
c, err := ipld.RawdataToCid(codec, raw, mh)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
|
||||
tx.iplds <- models.IPLDModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
Key: prefixedKey,
|
||||
Data: raw,
|
||||
}
|
||||
return c.String(), prefixedKey, err
|
||||
}
|
||||
79
statediff/indexer/database/dump/config.go
Normal file
79
statediff/indexer/database/dump/config.go
Normal file
@ -0,0 +1,79 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package dump
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
)
|
||||
|
||||
// DumpType to explicitly type the dump destination
|
||||
type DumpType string
|
||||
|
||||
const (
|
||||
STDOUT = "Stdout"
|
||||
STDERR = "Stderr"
|
||||
DISCARD = "Discard"
|
||||
UNKNOWN = "Unknown"
|
||||
)
|
||||
|
||||
// ResolveDumpType resolves the dump type for the provided string
|
||||
func ResolveDumpType(str string) (DumpType, error) {
|
||||
switch strings.ToLower(str) {
|
||||
case "stdout", "out", "std out":
|
||||
return STDOUT, nil
|
||||
case "stderr", "err", "std err":
|
||||
return STDERR, nil
|
||||
case "discard", "void", "devnull", "dev null":
|
||||
return DISCARD, nil
|
||||
default:
|
||||
return UNKNOWN, fmt.Errorf("unrecognized dump type: %s", str)
|
||||
}
|
||||
}
|
||||
|
||||
// Config for data dump
|
||||
type Config struct {
|
||||
Dump io.WriteCloser
|
||||
}
|
||||
|
||||
// Type satisfies interfaces.Config
|
||||
func (c Config) Type() shared.DBType {
|
||||
return shared.DUMP
|
||||
}
|
||||
|
||||
// NewDiscardWriterCloser returns a discardWrapper wrapping io.Discard
|
||||
func NewDiscardWriterCloser() io.WriteCloser {
|
||||
return discardWrapper{blackhole: io.Discard}
|
||||
}
|
||||
|
||||
// discardWrapper wraps io.Discard with io.Closer
|
||||
type discardWrapper struct {
|
||||
blackhole io.Writer
|
||||
}
|
||||
|
||||
// Write satisfies io.Writer
|
||||
func (dw discardWrapper) Write(b []byte) (int, error) {
|
||||
return dw.blackhole.Write(b)
|
||||
}
|
||||
|
||||
// Close satisfies io.Closer
|
||||
func (dw discardWrapper) Close() error {
|
||||
return nil
|
||||
}
|
||||
535
statediff/indexer/database/dump/indexer.go
Normal file
535
statediff/indexer/database/dump/indexer.go
Normal file
@ -0,0 +1,535 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package dump
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
|
||||
ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
"github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/models"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
|
||||
)
|
||||
|
||||
var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
|
||||
|
||||
// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of a void
|
||||
type StateDiffIndexer struct {
|
||||
dump io.WriteCloser
|
||||
chainConfig *params.ChainConfig
|
||||
}
|
||||
|
||||
// NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer
|
||||
func NewStateDiffIndexer(chainConfig *params.ChainConfig, config Config) *StateDiffIndexer {
|
||||
return &StateDiffIndexer{
|
||||
dump: config.Dump,
|
||||
chainConfig: chainConfig,
|
||||
}
|
||||
}
|
||||
|
||||
// ReportDBMetrics has nothing to report for dump
|
||||
func (sdi *StateDiffIndexer) ReportDBMetrics(time.Duration, <-chan bool) {}
|
||||
|
||||
// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
|
||||
// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
|
||||
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
|
||||
start, t := time.Now(), time.Now()
|
||||
blockHash := block.Hash()
|
||||
blockHashStr := blockHash.String()
|
||||
height := block.NumberU64()
|
||||
traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
|
||||
transactions := block.Transactions()
|
||||
// Derive any missing fields
|
||||
if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Generate the block iplds
|
||||
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
|
||||
}
|
||||
|
||||
if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) {
|
||||
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
|
||||
}
|
||||
if len(txTrieNodes) != len(rctTrieNodes) {
|
||||
return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
|
||||
}
|
||||
|
||||
// Calculate reward
|
||||
var reward *big.Int
|
||||
// in PoA networks block reward is 0
|
||||
if sdi.chainConfig.Clique != nil {
|
||||
reward = big.NewInt(0)
|
||||
} else {
|
||||
reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts)
|
||||
}
|
||||
t = time.Now()
|
||||
|
||||
blockTx := &BatchTx{
|
||||
BlockNumber: block.Number().String(),
|
||||
dump: sdi.dump,
|
||||
iplds: make(chan models.IPLDModel),
|
||||
quit: make(chan struct{}),
|
||||
ipldCache: models.IPLDBatch{},
|
||||
submit: func(self *BatchTx, err error) error {
|
||||
close(self.quit)
|
||||
close(self.iplds)
|
||||
tDiff := time.Since(t)
|
||||
metrics.IndexerMetrics.StateStoreCodeProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
if err := self.flush(); err != nil {
|
||||
traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
|
||||
log.Debug(traceMsg)
|
||||
return err
|
||||
}
|
||||
tDiff = time.Since(t)
|
||||
metrics.IndexerMetrics.PostgresCommitTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
|
||||
traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
|
||||
log.Debug(traceMsg)
|
||||
return err
|
||||
},
|
||||
}
|
||||
go blockTx.cache()
|
||||
|
||||
tDiff := time.Since(t)
|
||||
metrics.IndexerMetrics.FreePostgresTimer.Update(tDiff)
|
||||
|
||||
traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
|
||||
// Publish and index header, collect headerID
|
||||
var headerID string
|
||||
headerID, err = sdi.processHeader(blockTx, block.Header(), headerNode, reward, totalDifficulty)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tDiff = time.Since(t)
|
||||
metrics.IndexerMetrics.HeaderProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
// Publish and index uncles
|
||||
err = sdi.processUncles(blockTx, headerID, block.Number(), uncleNodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tDiff = time.Since(t)
|
||||
metrics.IndexerMetrics.UncleProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
// Publish and index receipts and txs
|
||||
err = sdi.processReceiptsAndTxs(blockTx, processArgs{
|
||||
headerID: headerID,
|
||||
blockNumber: block.Number(),
|
||||
receipts: receipts,
|
||||
txs: transactions,
|
||||
rctNodes: rctNodes,
|
||||
rctTrieNodes: rctTrieNodes,
|
||||
txNodes: txNodes,
|
||||
txTrieNodes: txTrieNodes,
|
||||
logTrieNodes: logTrieNodes,
|
||||
logLeafNodeCIDs: logLeafNodeCIDs,
|
||||
rctLeafNodeCIDs: rctLeafNodeCIDs,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tDiff = time.Since(t)
|
||||
metrics.IndexerMetrics.TxAndRecProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
|
||||
return blockTx, err
|
||||
}
|
||||
|
||||
// processHeader publishes and indexes a header IPLD in Postgres
|
||||
// it returns the headerID
|
||||
func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode node.Node, reward, td *big.Int) (string, error) {
|
||||
tx.cacheIPLD(headerNode)
|
||||
|
||||
headerID := header.Hash().String()
|
||||
mod := models.HeaderModel{
|
||||
CID: headerNode.Cid().String(),
|
||||
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
|
||||
ParentHash: header.ParentHash.String(),
|
||||
BlockNumber: header.Number.String(),
|
||||
BlockHash: headerID,
|
||||
TotalDifficulty: td.String(),
|
||||
Reward: reward.String(),
|
||||
Bloom: header.Bloom.Bytes(),
|
||||
StateRoot: header.Root.String(),
|
||||
RctRoot: header.ReceiptHash.String(),
|
||||
TxRoot: header.TxHash.String(),
|
||||
UncleRoot: header.UncleHash.String(),
|
||||
Timestamp: header.Time,
|
||||
Coinbase: header.Coinbase.String(),
|
||||
}
|
||||
_, err := fmt.Fprintf(sdi.dump, "%+v\r\n", mod)
|
||||
return headerID, err
|
||||
}
|
||||
|
||||
// processUncles publishes and indexes uncle IPLDs in Postgres
|
||||
func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, uncleNodes []*ipld2.EthHeader) error {
|
||||
// publish and index uncles
|
||||
for _, uncleNode := range uncleNodes {
|
||||
tx.cacheIPLD(uncleNode)
|
||||
var uncleReward *big.Int
|
||||
// in PoA networks uncle reward is 0
|
||||
if sdi.chainConfig.Clique != nil {
|
||||
uncleReward = big.NewInt(0)
|
||||
} else {
|
||||
uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncleNode.Number.Uint64())
|
||||
}
|
||||
uncle := models.UncleModel{
|
||||
BlockNumber: blockNumber.String(),
|
||||
HeaderID: headerID,
|
||||
CID: uncleNode.Cid().String(),
|
||||
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
|
||||
ParentHash: uncleNode.ParentHash.String(),
|
||||
BlockHash: uncleNode.Hash().String(),
|
||||
Reward: uncleReward.String(),
|
||||
}
|
||||
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", uncle); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// processArgs bundles arguments to processReceiptsAndTxs
|
||||
type processArgs struct {
|
||||
headerID string
|
||||
blockNumber *big.Int
|
||||
receipts types.Receipts
|
||||
txs types.Transactions
|
||||
rctNodes []*ipld2.EthReceipt
|
||||
rctTrieNodes []*ipld2.EthRctTrie
|
||||
txNodes []*ipld2.EthTx
|
||||
txTrieNodes []*ipld2.EthTxTrie
|
||||
logTrieNodes [][]node.Node
|
||||
logLeafNodeCIDs [][]cid.Cid
|
||||
rctLeafNodeCIDs []cid.Cid
|
||||
}
|
||||
|
||||
// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
|
||||
func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error {
|
||||
// Process receipts and txs
|
||||
signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
|
||||
for i, receipt := range args.receipts {
|
||||
for _, logTrieNode := range args.logTrieNodes[i] {
|
||||
tx.cacheIPLD(logTrieNode)
|
||||
}
|
||||
txNode := args.txNodes[i]
|
||||
tx.cacheIPLD(txNode)
|
||||
|
||||
// Indexing
|
||||
// index tx
|
||||
trx := args.txs[i]
|
||||
trxID := trx.Hash().String()
|
||||
|
||||
var val string
|
||||
if trx.Value() != nil {
|
||||
val = trx.Value().String()
|
||||
}
|
||||
|
||||
// derive sender for the tx that corresponds with this receipt
|
||||
from, err := types.Sender(signer, trx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deriving tx sender: %v", err)
|
||||
}
|
||||
txModel := models.TxModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
HeaderID: args.headerID,
|
||||
Dst: shared.HandleZeroAddrPointer(trx.To()),
|
||||
Src: shared.HandleZeroAddr(from),
|
||||
TxHash: trxID,
|
||||
Index: int64(i),
|
||||
Data: trx.Data(),
|
||||
CID: txNode.Cid().String(),
|
||||
MhKey: shared.MultihashKeyFromCID(txNode.Cid()),
|
||||
Type: trx.Type(),
|
||||
Value: val,
|
||||
}
|
||||
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", txModel); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// index access list if this is one
|
||||
for j, accessListElement := range trx.AccessList() {
|
||||
storageKeys := make([]string, len(accessListElement.StorageKeys))
|
||||
for k, storageKey := range accessListElement.StorageKeys {
|
||||
storageKeys[k] = storageKey.Hex()
|
||||
}
|
||||
accessListElementModel := models.AccessListElementModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
TxID: trxID,
|
||||
Index: int64(j),
|
||||
Address: accessListElement.Address.Hex(),
|
||||
StorageKeys: storageKeys,
|
||||
}
|
||||
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", accessListElementModel); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// this is the contract address if this receipt is for a contract creation tx
|
||||
contract := shared.HandleZeroAddr(receipt.ContractAddress)
|
||||
var contractHash string
|
||||
if contract != "" {
|
||||
contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
|
||||
}
|
||||
|
||||
// index the receipt
|
||||
if !args.rctLeafNodeCIDs[i].Defined() {
|
||||
return fmt.Errorf("invalid receipt leaf node cid")
|
||||
}
|
||||
|
||||
rctModel := &models.ReceiptModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
HeaderID: args.headerID,
|
||||
TxID: trxID,
|
||||
Contract: contract,
|
||||
ContractHash: contractHash,
|
||||
LeafCID: args.rctLeafNodeCIDs[i].String(),
|
||||
LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]),
|
||||
LogRoot: args.rctNodes[i].LogRoot.String(),
|
||||
}
|
||||
if len(receipt.PostState) == 0 {
|
||||
rctModel.PostStatus = receipt.Status
|
||||
} else {
|
||||
rctModel.PostState = common.Bytes2Hex(receipt.PostState)
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", rctModel); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logDataSet := make([]*models.LogsModel, len(receipt.Logs))
|
||||
for idx, l := range receipt.Logs {
|
||||
topicSet := make([]string, 4)
|
||||
for ti, topic := range l.Topics {
|
||||
topicSet[ti] = topic.Hex()
|
||||
}
|
||||
|
||||
if !args.logLeafNodeCIDs[i][idx].Defined() {
|
||||
return fmt.Errorf("invalid log cid")
|
||||
}
|
||||
|
||||
logDataSet[idx] = &models.LogsModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
HeaderID: args.headerID,
|
||||
ReceiptID: trxID,
|
||||
Address: l.Address.String(),
|
||||
Index: int64(l.Index),
|
||||
Data: l.Data,
|
||||
LeafCID: args.logLeafNodeCIDs[i][idx].String(),
|
||||
LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]),
|
||||
Topic0: topicSet[0],
|
||||
Topic1: topicSet[1],
|
||||
Topic2: topicSet[2],
|
||||
Topic3: topicSet[3],
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", logDataSet); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// publish trie nodes, these aren't indexed directly
|
||||
for i, n := range args.txTrieNodes {
|
||||
tx.cacheIPLD(n)
|
||||
tx.cacheIPLD(args.rctTrieNodes[i])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD sql
|
||||
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error {
|
||||
tx, ok := batch.(*BatchTx)
|
||||
if !ok {
|
||||
return fmt.Errorf("dump: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
|
||||
}
|
||||
// publish the state node
|
||||
var stateModel models.StateNodeModel
|
||||
if stateNode.NodeType == sdtypes.Removed {
|
||||
// short circuit if it is a Removed node
|
||||
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
|
||||
stateModel = models.StateNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
Path: stateNode.Path,
|
||||
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
|
||||
CID: shared.RemovedNodeStateCID,
|
||||
MhKey: shared.RemovedNodeMhKey,
|
||||
NodeType: stateNode.NodeType.Int(),
|
||||
}
|
||||
} else {
|
||||
stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error generating and cacheing state node IPLD: %v", err)
|
||||
}
|
||||
stateModel = models.StateNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
Path: stateNode.Path,
|
||||
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
|
||||
CID: stateCIDStr,
|
||||
MhKey: stateMhKey,
|
||||
NodeType: stateNode.NodeType.Int(),
|
||||
}
|
||||
}
|
||||
|
||||
// index the state node, collect the stateID to reference by FK
|
||||
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", stateModel); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// if we have a leaf, decode and index the account data
|
||||
if stateNode.NodeType == sdtypes.Leaf {
|
||||
var i []interface{}
|
||||
if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil {
|
||||
return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
|
||||
}
|
||||
if len(i) != 2 {
|
||||
return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements")
|
||||
}
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
|
||||
return fmt.Errorf("error decoding state account rlp: %s", err.Error())
|
||||
}
|
||||
accountModel := models.StateAccountModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
StatePath: stateNode.Path,
|
||||
Balance: account.Balance.String(),
|
||||
Nonce: account.Nonce,
|
||||
CodeHash: account.CodeHash,
|
||||
StorageRoot: account.Root.String(),
|
||||
}
|
||||
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", accountModel); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// if there are any storage nodes associated with this node, publish and index them
|
||||
for _, storageNode := range stateNode.StorageNodes {
|
||||
if storageNode.NodeType == sdtypes.Removed {
|
||||
// short circuit if it is a Removed node
|
||||
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
|
||||
storageModel := models.StorageNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
StatePath: stateNode.Path,
|
||||
Path: storageNode.Path,
|
||||
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
|
||||
CID: shared.RemovedNodeStorageCID,
|
||||
MhKey: shared.RemovedNodeMhKey,
|
||||
NodeType: storageNode.NodeType.Int(),
|
||||
}
|
||||
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", storageModel); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
storageCIDStr, storageMhKey, err := tx.cacheRaw(ipld2.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err)
|
||||
}
|
||||
storageModel := models.StorageNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
StatePath: stateNode.Path,
|
||||
Path: storageNode.Path,
|
||||
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
|
||||
CID: storageCIDStr,
|
||||
MhKey: storageMhKey,
|
||||
NodeType: storageNode.NodeType.Int(),
|
||||
}
|
||||
if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", storageModel); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PushCodeAndCodeHash publishes code and codehash pairs to the ipld sql
|
||||
func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error {
|
||||
tx, ok := batch.(*BatchTx)
|
||||
if !ok {
|
||||
return fmt.Errorf("dump: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
|
||||
}
|
||||
// codec doesn't matter since db key is multihash-based
|
||||
mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deriving multihash key from codehash: %v", err)
|
||||
}
|
||||
tx.cacheDirect(mhKey, codeAndCodeHash.Code)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close satisfies io.Closer
|
||||
func (sdi *StateDiffIndexer) Close() error {
|
||||
return sdi.dump.Close()
|
||||
}
|
||||
|
||||
// LoadWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
|
||||
func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// InsertWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
|
||||
func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
|
||||
func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
|
||||
func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearWatchedAddresses satisfies the interfaces.StateDiffIndexer interface
|
||||
func (sdi *StateDiffIndexer) ClearWatchedAddresses() error {
|
||||
return nil
|
||||
}
|
||||
29
statediff/indexer/database/file/batch_tx.go
Normal file
29
statediff/indexer/database/file/batch_tx.go
Normal file
@ -0,0 +1,29 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file
|
||||
|
||||
// BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration
|
||||
type BatchTx struct {
|
||||
BlockNumber string
|
||||
|
||||
submit func(blockTx *BatchTx, err error) error
|
||||
}
|
||||
|
||||
// Submit satisfies indexer.AtomicTx
|
||||
func (tx *BatchTx) Submit(err error) error {
|
||||
return tx.submit(tx, err)
|
||||
}
|
||||
84
statediff/indexer/database/file/config.go
Normal file
84
statediff/indexer/database/file/config.go
Normal file
@ -0,0 +1,84 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
)
|
||||
|
||||
// FileMode to explicitly type the mode of file writer we are using
|
||||
type FileMode string
|
||||
|
||||
const (
|
||||
CSV FileMode = "CSV"
|
||||
SQL FileMode = "SQL"
|
||||
Unknown FileMode = "Unknown"
|
||||
)
|
||||
|
||||
// ResolveFileMode resolves a FileMode from a provided string
|
||||
func ResolveFileMode(str string) (FileMode, error) {
|
||||
switch strings.ToLower(str) {
|
||||
case "csv":
|
||||
return CSV, nil
|
||||
case "sql":
|
||||
return SQL, nil
|
||||
default:
|
||||
return Unknown, fmt.Errorf("unrecognized file type string: %s", str)
|
||||
}
|
||||
}
|
||||
|
||||
// Config holds params for writing out CSV or SQL files
|
||||
type Config struct {
|
||||
Mode FileMode
|
||||
OutputDir string
|
||||
FilePath string
|
||||
WatchedAddressesFilePath string
|
||||
NodeInfo node.Info
|
||||
}
|
||||
|
||||
// Type satisfies interfaces.Config
|
||||
func (c Config) Type() shared.DBType {
|
||||
return shared.FILE
|
||||
}
|
||||
|
||||
var nodeInfo = node.Info{
|
||||
GenesisBlock: "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3",
|
||||
NetworkID: "1",
|
||||
ChainID: 1,
|
||||
ID: "mockNodeID",
|
||||
ClientName: "go-ethereum",
|
||||
}
|
||||
|
||||
// CSVTestConfig config for unit tests
|
||||
var CSVTestConfig = Config{
|
||||
Mode: CSV,
|
||||
OutputDir: "./statediffing_test",
|
||||
WatchedAddressesFilePath: "./statediffing_watched_addresses_test_file.csv",
|
||||
NodeInfo: nodeInfo,
|
||||
}
|
||||
|
||||
// SQLTestConfig config for unit tests
|
||||
var SQLTestConfig = Config{
|
||||
Mode: SQL,
|
||||
FilePath: "./statediffing_test_file.sql",
|
||||
WatchedAddressesFilePath: "./statediffing_watched_addresses_test_file.sql",
|
||||
NodeInfo: nodeInfo,
|
||||
}
|
||||
118
statediff/indexer/database/file/csv_indexer_legacy_test.go
Normal file
118
statediff/indexer/database/file/csv_indexer_legacy_test.go
Normal file
@ -0,0 +1,118 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2022 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/file/types"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
|
||||
)
|
||||
|
||||
const dbDirectory = "/file_indexer"
|
||||
const pgCopyStatement = `COPY %s FROM '%s' CSV`
|
||||
|
||||
func setupLegacyCSVIndexer(t *testing.T) {
|
||||
if _, err := os.Stat(file.CSVTestConfig.OutputDir); !errors.Is(err, os.ErrNotExist) {
|
||||
err := os.RemoveAll(file.CSVTestConfig.OutputDir)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.CSVTestConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err = postgres.SetupSQLXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func setupLegacyCSV(t *testing.T) {
|
||||
setupLegacyCSVIndexer(t)
|
||||
test.SetupLegacyTestData(t, ind)
|
||||
}
|
||||
|
||||
func dumpCSVFileData(t *testing.T) {
|
||||
outputDir := filepath.Join(dbDirectory, file.CSVTestConfig.OutputDir)
|
||||
workingDir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
|
||||
localOutputDir := filepath.Join(workingDir, file.CSVTestConfig.OutputDir)
|
||||
|
||||
for _, tbl := range file.Tables {
|
||||
err := test_helpers.DedupFile(file.TableFilePath(localOutputDir, tbl.Name))
|
||||
require.NoError(t, err)
|
||||
|
||||
var stmt string
|
||||
varcharColumns := tbl.VarcharColumns()
|
||||
if len(varcharColumns) > 0 {
|
||||
stmt = fmt.Sprintf(
|
||||
pgCopyStatement+" FORCE NOT NULL %s",
|
||||
tbl.Name,
|
||||
file.TableFilePath(outputDir, tbl.Name),
|
||||
strings.Join(varcharColumns, ", "),
|
||||
)
|
||||
} else {
|
||||
stmt = fmt.Sprintf(pgCopyStatement, tbl.Name, file.TableFilePath(outputDir, tbl.Name))
|
||||
}
|
||||
|
||||
_, err = db.Exec(context.Background(), stmt)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func resetAndDumpWatchedAddressesCSVFileData(t *testing.T) {
|
||||
test_helpers.TearDownDB(t, db)
|
||||
|
||||
outputFilePath := filepath.Join(dbDirectory, file.CSVTestConfig.WatchedAddressesFilePath)
|
||||
stmt := fmt.Sprintf(pgCopyStatement, types.TableWatchedAddresses.Name, outputFilePath)
|
||||
|
||||
_, err = db.Exec(context.Background(), stmt)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func tearDownCSV(t *testing.T) {
|
||||
test_helpers.TearDownDB(t, db)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
require.NoError(t, os.RemoveAll(file.CSVTestConfig.OutputDir))
|
||||
|
||||
if err := os.Remove(file.CSVTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyCSVFileIndexer(t *testing.T) {
|
||||
t.Run("Publish and index header IPLDs", func(t *testing.T) {
|
||||
setupLegacyCSV(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestLegacyIndexer(t, db)
|
||||
})
|
||||
}
|
||||
255
statediff/indexer/database/file/csv_indexer_test.go
Normal file
255
statediff/indexer/database/file/csv_indexer_test.go
Normal file
@ -0,0 +1,255 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2022 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math/big"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
)
|
||||
|
||||
func setupCSVIndexer(t *testing.T) {
|
||||
file.CSVTestConfig.OutputDir = "./statediffing_test"
|
||||
|
||||
if _, err := os.Stat(file.CSVTestConfig.OutputDir); !errors.Is(err, os.ErrNotExist) {
|
||||
err := os.RemoveAll(file.CSVTestConfig.OutputDir)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(file.CSVTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) {
|
||||
err := os.Remove(file.CSVTestConfig.WatchedAddressesFilePath)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.CSVTestConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err = postgres.SetupSQLXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func setupCSV(t *testing.T) {
|
||||
setupCSVIndexer(t)
|
||||
test.SetupTestData(t, ind)
|
||||
}
|
||||
|
||||
func setupCSVNonCanonical(t *testing.T) {
|
||||
setupCSVIndexer(t)
|
||||
test.SetupTestDataNonCanonical(t, ind)
|
||||
}
|
||||
|
||||
func TestCSVFileIndexer(t *testing.T) {
|
||||
t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
|
||||
setupCSV(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexHeaderIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
|
||||
setupCSV(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexTransactionIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
|
||||
setupCSV(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexLogIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
|
||||
setupCSV(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexReceiptIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
|
||||
setupCSV(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexStateIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
|
||||
setupCSV(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexStorageIPLDs(t, db)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCSVFileIndexerNonCanonical(t *testing.T) {
|
||||
t.Run("Publish and index header", func(t *testing.T) {
|
||||
setupCSVNonCanonical(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexHeaderNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index transactions", func(t *testing.T) {
|
||||
setupCSVNonCanonical(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index receipts", func(t *testing.T) {
|
||||
setupCSVNonCanonical(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index logs", func(t *testing.T) {
|
||||
setupCSVNonCanonical(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexLogsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index state nodes", func(t *testing.T) {
|
||||
setupCSVNonCanonical(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexStateNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index storage nodes", func(t *testing.T) {
|
||||
setupCSVNonCanonical(t)
|
||||
dumpCSVFileData(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
test.TestPublishAndIndexStorageNonCanonical(t, db)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCSVFileWatchAddressMethods(t *testing.T) {
|
||||
setupCSVIndexer(t)
|
||||
defer tearDownCSV(t)
|
||||
|
||||
t.Run("Load watched addresses (empty table)", func(t *testing.T) {
|
||||
test.TestLoadEmptyWatchedAddresses(t, ind)
|
||||
})
|
||||
|
||||
t.Run("Insert watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetInsertWatchedAddressesArgs()
|
||||
err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesCSVFileData(t)
|
||||
|
||||
test.TestInsertWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
|
||||
args := mocks.GetInsertAlreadyWatchedAddressesArgs()
|
||||
err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesCSVFileData(t)
|
||||
|
||||
test.TestInsertAlreadyWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Remove watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetRemoveWatchedAddressesArgs()
|
||||
err = ind.RemoveWatchedAddresses(args)
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesCSVFileData(t)
|
||||
|
||||
test.TestRemoveWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
|
||||
args := mocks.GetRemoveNonWatchedAddressesArgs()
|
||||
err = ind.RemoveWatchedAddresses(args)
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesCSVFileData(t)
|
||||
|
||||
test.TestRemoveNonWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Set watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetSetWatchedAddressesArgs()
|
||||
err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesCSVFileData(t)
|
||||
|
||||
test.TestSetWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
|
||||
args := mocks.GetSetAlreadyWatchedAddressesArgs()
|
||||
err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesCSVFileData(t)
|
||||
|
||||
test.TestSetAlreadyWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Load watched addresses", func(t *testing.T) {
|
||||
test.TestLoadWatchedAddresses(t, ind)
|
||||
})
|
||||
|
||||
t.Run("Clear watched addresses", func(t *testing.T) {
|
||||
err = ind.ClearWatchedAddresses()
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesCSVFileData(t)
|
||||
|
||||
test.TestClearWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
|
||||
err = ind.ClearWatchedAddresses()
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesCSVFileData(t)
|
||||
|
||||
test.TestClearEmptyWatchedAddresses(t, db)
|
||||
})
|
||||
}
|
||||
457
statediff/indexer/database/file/csv_writer.go
Normal file
457
statediff/indexer/database/file/csv_writer.go
Normal file
@ -0,0 +1,457 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2022 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
dshelp "github.com/ipfs/go-ipfs-ds-help"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
"github.com/thoas/go-funk"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/file/types"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/models"
|
||||
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
|
||||
)
|
||||
|
||||
var (
|
||||
Tables = []*types.Table{
|
||||
&types.TableIPLDBlock,
|
||||
&types.TableNodeInfo,
|
||||
&types.TableHeader,
|
||||
&types.TableStateNode,
|
||||
&types.TableStorageNode,
|
||||
&types.TableUncle,
|
||||
&types.TableTransaction,
|
||||
&types.TableAccessListElement,
|
||||
&types.TableReceipt,
|
||||
&types.TableLog,
|
||||
&types.TableStateAccount,
|
||||
}
|
||||
)
|
||||
|
||||
type tableRow struct {
|
||||
table types.Table
|
||||
values []interface{}
|
||||
}
|
||||
|
||||
type CSVWriter struct {
|
||||
// dir containing output files
|
||||
dir string
|
||||
|
||||
writers fileWriters
|
||||
watchedAddressesWriter fileWriter
|
||||
|
||||
rows chan tableRow
|
||||
flushChan chan struct{}
|
||||
flushFinished chan struct{}
|
||||
quitChan chan struct{}
|
||||
doneChan chan struct{}
|
||||
}
|
||||
|
||||
type fileWriter struct {
|
||||
*csv.Writer
|
||||
file *os.File
|
||||
}
|
||||
|
||||
// fileWriters wraps the file writers for each output table
|
||||
type fileWriters map[string]fileWriter
|
||||
|
||||
func newFileWriter(path string) (ret fileWriter, err error) {
|
||||
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ret = fileWriter{
|
||||
Writer: csv.NewWriter(file),
|
||||
file: file,
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func makeFileWriters(dir string, tables []*types.Table) (fileWriters, error) {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writers := fileWriters{}
|
||||
for _, tbl := range tables {
|
||||
w, err := newFileWriter(TableFilePath(dir, tbl.Name))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writers[tbl.Name] = w
|
||||
}
|
||||
return writers, nil
|
||||
}
|
||||
|
||||
func (tx fileWriters) write(tbl *types.Table, args ...interface{}) error {
|
||||
row := tbl.ToCsvRow(args...)
|
||||
return tx[tbl.Name].Write(row)
|
||||
}
|
||||
|
||||
func (tx fileWriters) close() error {
|
||||
for _, w := range tx {
|
||||
err := w.file.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tx fileWriters) flush() error {
|
||||
for _, w := range tx {
|
||||
w.Flush()
|
||||
if err := w.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewCSVWriter(path string, watchedAddressesFilePath string) (*CSVWriter, error) {
|
||||
if err := os.MkdirAll(path, 0777); err != nil {
|
||||
return nil, fmt.Errorf("unable to make MkdirAll for path: %s err: %s", path, err)
|
||||
}
|
||||
|
||||
writers, err := makeFileWriters(path, Tables)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
watchedAddressesWriter, err := newFileWriter(watchedAddressesFilePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
csvWriter := &CSVWriter{
|
||||
writers: writers,
|
||||
watchedAddressesWriter: watchedAddressesWriter,
|
||||
dir: path,
|
||||
rows: make(chan tableRow),
|
||||
flushChan: make(chan struct{}),
|
||||
flushFinished: make(chan struct{}),
|
||||
quitChan: make(chan struct{}),
|
||||
doneChan: make(chan struct{}),
|
||||
}
|
||||
return csvWriter, nil
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) Loop() {
|
||||
go func() {
|
||||
defer close(csw.doneChan)
|
||||
for {
|
||||
select {
|
||||
case row := <-csw.rows:
|
||||
err := csw.writers.write(&row.table, row.values...)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("error writing csv buffer: %v", err))
|
||||
}
|
||||
case <-csw.quitChan:
|
||||
if err := csw.writers.flush(); err != nil {
|
||||
panic(fmt.Sprintf("error writing csv buffer to file: %v", err))
|
||||
}
|
||||
return
|
||||
case <-csw.flushChan:
|
||||
if err := csw.writers.flush(); err != nil {
|
||||
panic(fmt.Sprintf("error writing csv buffer to file: %v", err))
|
||||
}
|
||||
csw.flushFinished <- struct{}{}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Flush sends a flush signal to the looping process
|
||||
func (csw *CSVWriter) Flush() {
|
||||
csw.flushChan <- struct{}{}
|
||||
<-csw.flushFinished
|
||||
}
|
||||
|
||||
func TableFilePath(dir, name string) string { return filepath.Join(dir, name+".csv") }
|
||||
|
||||
// Close satisfies io.Closer
|
||||
func (csw *CSVWriter) Close() error {
|
||||
close(csw.quitChan)
|
||||
<-csw.doneChan
|
||||
close(csw.rows)
|
||||
close(csw.flushChan)
|
||||
close(csw.flushFinished)
|
||||
return csw.writers.close()
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertNode(node nodeinfo.Info) {
|
||||
var values []interface{}
|
||||
values = append(values, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID)
|
||||
csw.rows <- tableRow{types.TableNodeInfo, values}
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertIPLD(ipld models.IPLDModel) {
|
||||
var values []interface{}
|
||||
values = append(values, ipld.BlockNumber, ipld.Key, ipld.Data)
|
||||
csw.rows <- tableRow{types.TableIPLDBlock, values}
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertIPLDDirect(blockNumber, key string, value []byte) {
|
||||
csw.upsertIPLD(models.IPLDModel{
|
||||
BlockNumber: blockNumber,
|
||||
Key: key,
|
||||
Data: value,
|
||||
})
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertIPLDNode(blockNumber string, i node.Node) {
|
||||
csw.upsertIPLD(models.IPLDModel{
|
||||
BlockNumber: blockNumber,
|
||||
Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(),
|
||||
Data: i.RawData(),
|
||||
})
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error) {
|
||||
c, err := ipld.RawdataToCid(codec, raw, mh)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
|
||||
csw.upsertIPLD(models.IPLDModel{
|
||||
BlockNumber: blockNumber,
|
||||
Key: prefixedKey,
|
||||
Data: raw,
|
||||
})
|
||||
return c.String(), prefixedKey, err
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) {
|
||||
var values []interface{}
|
||||
values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
|
||||
header.TotalDifficulty, header.NodeID, header.Reward, header.StateRoot, header.TxRoot,
|
||||
header.RctRoot, header.UncleRoot, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.MhKey, 1, header.Coinbase)
|
||||
csw.rows <- tableRow{types.TableHeader, values}
|
||||
metrics.IndexerMetrics.BlocksCounter.Inc(1)
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertUncleCID(uncle models.UncleModel) {
|
||||
var values []interface{}
|
||||
values = append(values, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID,
|
||||
uncle.Reward, uncle.MhKey)
|
||||
csw.rows <- tableRow{types.TableUncle, values}
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertTransactionCID(transaction models.TxModel) {
|
||||
var values []interface{}
|
||||
values = append(values, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
|
||||
transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value)
|
||||
csw.rows <- tableRow{types.TableTransaction, values}
|
||||
metrics.IndexerMetrics.TransactionsCounter.Inc(1)
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertAccessListElement(accessListElement models.AccessListElementModel) {
|
||||
var values []interface{}
|
||||
values = append(values, accessListElement.BlockNumber, accessListElement.TxID, accessListElement.Index, accessListElement.Address, accessListElement.StorageKeys)
|
||||
csw.rows <- tableRow{types.TableAccessListElement, values}
|
||||
metrics.IndexerMetrics.AccessListEntriesCounter.Inc(1)
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) {
|
||||
var values []interface{}
|
||||
values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey,
|
||||
rct.PostState, rct.PostStatus, rct.LogRoot)
|
||||
csw.rows <- tableRow{types.TableReceipt, values}
|
||||
metrics.IndexerMetrics.ReceiptsCounter.Inc(1)
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertLogCID(logs []*models.LogsModel) {
|
||||
for _, l := range logs {
|
||||
var values []interface{}
|
||||
values = append(values, l.BlockNumber, l.HeaderID, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0,
|
||||
l.Topic1, l.Topic2, l.Topic3, l.Data)
|
||||
csw.rows <- tableRow{types.TableLog, values}
|
||||
metrics.IndexerMetrics.LogsCounter.Inc(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertStateCID(stateNode models.StateNodeModel) {
|
||||
var stateKey string
|
||||
if stateNode.StateKey != nullHash.String() {
|
||||
stateKey = stateNode.StateKey
|
||||
}
|
||||
|
||||
var values []interface{}
|
||||
values = append(values, stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path,
|
||||
stateNode.NodeType, true, stateNode.MhKey)
|
||||
csw.rows <- tableRow{types.TableStateNode, values}
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertStateAccount(stateAccount models.StateAccountModel) {
|
||||
var values []interface{}
|
||||
values = append(values, stateAccount.BlockNumber, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance,
|
||||
strconv.FormatUint(stateAccount.Nonce, 10), stateAccount.CodeHash, stateAccount.StorageRoot)
|
||||
csw.rows <- tableRow{types.TableStateAccount, values}
|
||||
}
|
||||
|
||||
func (csw *CSVWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
|
||||
var storageKey string
|
||||
if storageCID.StorageKey != nullHash.String() {
|
||||
storageKey = storageCID.StorageKey
|
||||
}
|
||||
|
||||
var values []interface{}
|
||||
values = append(values, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID,
|
||||
storageCID.Path, storageCID.NodeType, true, storageCID.MhKey)
|
||||
csw.rows <- tableRow{types.TableStorageNode, values}
|
||||
}
|
||||
|
||||
// LoadWatchedAddresses loads watched addresses from a file
|
||||
func (csw *CSVWriter) loadWatchedAddresses() ([]common.Address, error) {
|
||||
watchedAddressesFilePath := csw.watchedAddressesWriter.file.Name()
|
||||
// load csv rows from watched addresses file
|
||||
rows, err := loadWatchedAddressesRows(watchedAddressesFilePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// extract addresses from the csv rows
|
||||
watchedAddresses := funk.Map(rows, func(row []string) common.Address {
|
||||
// first column is for address in eth_meta.watched_addresses
|
||||
addressString := row[0]
|
||||
|
||||
return common.HexToAddress(addressString)
|
||||
}).([]common.Address)
|
||||
|
||||
return watchedAddresses, nil
|
||||
}
|
||||
|
||||
// InsertWatchedAddresses inserts the given addresses in a file
|
||||
func (csw *CSVWriter) insertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
|
||||
// load csv rows from watched addresses file
|
||||
watchedAddresses, err := csw.loadWatchedAddresses()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// append rows for new addresses to existing csv file
|
||||
for _, arg := range args {
|
||||
// ignore if already watched
|
||||
if funk.Contains(watchedAddresses, common.HexToAddress(arg.Address)) {
|
||||
continue
|
||||
}
|
||||
|
||||
var values []interface{}
|
||||
values = append(values, arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0")
|
||||
row := types.TableWatchedAddresses.ToCsvRow(values...)
|
||||
|
||||
// writing directly instead of using rows channel as it needs to be flushed immediately
|
||||
err = csw.watchedAddressesWriter.Write(row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// watched addresses need to be flushed immediately to the file to keep them in sync with in-memory watched addresses
|
||||
csw.watchedAddressesWriter.Flush()
|
||||
err = csw.watchedAddressesWriter.Error()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveWatchedAddresses removes the given watched addresses from a file
|
||||
func (csw *CSVWriter) removeWatchedAddresses(args []sdtypes.WatchAddressArg) error {
|
||||
// load csv rows from watched addresses file
|
||||
watchedAddressesFilePath := csw.watchedAddressesWriter.file.Name()
|
||||
rows, err := loadWatchedAddressesRows(watchedAddressesFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// get rid of rows having addresses to be removed
|
||||
filteredRows := funk.Filter(rows, func(row []string) bool {
|
||||
return !funk.Contains(args, func(arg sdtypes.WatchAddressArg) bool {
|
||||
// Compare first column in table for address
|
||||
return arg.Address == row[0]
|
||||
})
|
||||
}).([][]string)
|
||||
|
||||
return dumpWatchedAddressesRows(csw.watchedAddressesWriter, filteredRows)
|
||||
}
|
||||
|
||||
// SetWatchedAddresses clears and inserts the given addresses in a file
|
||||
func (csw *CSVWriter) setWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
|
||||
var rows [][]string
|
||||
for _, arg := range args {
|
||||
row := types.TableWatchedAddresses.ToCsvRow(arg.Address, strconv.FormatUint(arg.CreatedAt, 10), currentBlockNumber.String(), "0")
|
||||
rows = append(rows, row)
|
||||
}
|
||||
|
||||
return dumpWatchedAddressesRows(csw.watchedAddressesWriter, rows)
|
||||
}
|
||||
|
||||
// loadCSVWatchedAddresses loads csv rows from the given file
|
||||
func loadWatchedAddressesRows(filePath string) ([][]string, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return [][]string{}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("error opening watched addresses file: %v", err)
|
||||
}
|
||||
|
||||
defer file.Close()
|
||||
reader := csv.NewReader(file)
|
||||
|
||||
return reader.ReadAll()
|
||||
}
|
||||
|
||||
// dumpWatchedAddressesRows dumps csv rows to the given file
|
||||
func dumpWatchedAddressesRows(watchedAddressesWriter fileWriter, filteredRows [][]string) error {
|
||||
file := watchedAddressesWriter.file
|
||||
file.Close()
|
||||
|
||||
file, err := os.Create(file.Name())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating watched addresses file: %v", err)
|
||||
}
|
||||
|
||||
watchedAddressesWriter.Writer = csv.NewWriter(file)
|
||||
watchedAddressesWriter.file = file
|
||||
|
||||
for _, row := range filteredRows {
|
||||
watchedAddressesWriter.Write(row)
|
||||
}
|
||||
|
||||
watchedAddressesWriter.Flush()
|
||||
|
||||
return nil
|
||||
}
|
||||
60
statediff/indexer/database/file/helpers.go
Normal file
60
statediff/indexer/database/file/helpers.go
Normal file
@ -0,0 +1,60 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file
|
||||
|
||||
import "bytes"
|
||||
|
||||
// formatPostgresStringArray parses an array of strings into the proper Postgres string representation of that array
|
||||
func formatPostgresStringArray(a []string) string {
|
||||
if a == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
if n := len(a); n > 0 {
|
||||
// There will be at least two curly brackets, 2*N bytes of quotes,
|
||||
// and N-1 bytes of delimiters.
|
||||
b := make([]byte, 1, 1+3*n)
|
||||
b[0] = '{'
|
||||
|
||||
b = appendArrayQuotedBytes(b, []byte(a[0]))
|
||||
for i := 1; i < n; i++ {
|
||||
b = append(b, ',')
|
||||
b = appendArrayQuotedBytes(b, []byte(a[i]))
|
||||
}
|
||||
|
||||
return string(append(b, '}'))
|
||||
}
|
||||
|
||||
return "{}"
|
||||
}
|
||||
|
||||
func appendArrayQuotedBytes(b, v []byte) []byte {
|
||||
b = append(b, '"')
|
||||
for {
|
||||
i := bytes.IndexAny(v, `"\`)
|
||||
if i < 0 {
|
||||
b = append(b, v...)
|
||||
break
|
||||
}
|
||||
if i > 0 {
|
||||
b = append(b, v[:i]...)
|
||||
}
|
||||
b = append(b, '\\', v[i])
|
||||
v = v[i+1:]
|
||||
}
|
||||
return append(b, '"')
|
||||
}
|
||||
573
statediff/indexer/database/file/indexer.go
Normal file
573
statediff/indexer/database/file/indexer.go
Normal file
@ -0,0 +1,573 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
"github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/models"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
|
||||
)
|
||||
|
||||
const defaultCSVOutputDir = "./statediff_output"
|
||||
const defaultSQLFilePath = "./statediff.sql"
|
||||
const defaultWatchedAddressesCSVFilePath = "./statediff-watched-addresses.csv"
|
||||
const defaultWatchedAddressesSQLFilePath = "./statediff-watched-addresses.sql"
|
||||
|
||||
const watchedAddressesInsert = "INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ('%s', '%d', '%d') ON CONFLICT (address) DO NOTHING;"
|
||||
|
||||
var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
|
||||
|
||||
// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of a void
|
||||
type StateDiffIndexer struct {
|
||||
fileWriter FileWriter
|
||||
chainConfig *params.ChainConfig
|
||||
nodeID string
|
||||
wg *sync.WaitGroup
|
||||
removedCacheFlag *uint32
|
||||
}
|
||||
|
||||
// NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer
|
||||
func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, config Config) (*StateDiffIndexer, error) {
|
||||
var err error
|
||||
var writer FileWriter
|
||||
|
||||
watchedAddressesFilePath := config.WatchedAddressesFilePath
|
||||
|
||||
switch config.Mode {
|
||||
case CSV:
|
||||
outputDir := config.OutputDir
|
||||
if outputDir == "" {
|
||||
outputDir = defaultCSVOutputDir
|
||||
}
|
||||
|
||||
if _, err := os.Stat(outputDir); !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("cannot create output directory, directory (%s) already exists", outputDir)
|
||||
}
|
||||
log.Info("Writing statediff CSV files to directory", "file", outputDir)
|
||||
|
||||
if watchedAddressesFilePath == "" {
|
||||
watchedAddressesFilePath = defaultWatchedAddressesCSVFilePath
|
||||
}
|
||||
log.Info("Writing watched addresses to file", "file", watchedAddressesFilePath)
|
||||
|
||||
writer, err = NewCSVWriter(outputDir, watchedAddressesFilePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case SQL:
|
||||
filePath := config.FilePath
|
||||
if filePath == "" {
|
||||
filePath = defaultSQLFilePath
|
||||
}
|
||||
if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("cannot create file, file (%s) already exists", filePath)
|
||||
}
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create file (%s), err: %v", filePath, err)
|
||||
}
|
||||
log.Info("Writing statediff SQL statements to file", "file", filePath)
|
||||
|
||||
if watchedAddressesFilePath == "" {
|
||||
watchedAddressesFilePath = defaultWatchedAddressesSQLFilePath
|
||||
}
|
||||
log.Info("Writing watched addresses to file", "file", watchedAddressesFilePath)
|
||||
|
||||
writer = NewSQLWriter(file, watchedAddressesFilePath)
|
||||
default:
|
||||
return nil, fmt.Errorf("unrecognized file mode: %s", config.Mode)
|
||||
}
|
||||
|
||||
wg := new(sync.WaitGroup)
|
||||
writer.Loop()
|
||||
writer.upsertNode(config.NodeInfo)
|
||||
|
||||
return &StateDiffIndexer{
|
||||
fileWriter: writer,
|
||||
chainConfig: chainConfig,
|
||||
nodeID: config.NodeInfo.ID,
|
||||
wg: wg,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ReportDBMetrics has nothing to report for dump
|
||||
func (sdi *StateDiffIndexer) ReportDBMetrics(time.Duration, <-chan bool) {}
|
||||
|
||||
// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
|
||||
// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
|
||||
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
|
||||
sdi.removedCacheFlag = new(uint32)
|
||||
start, t := time.Now(), time.Now()
|
||||
blockHash := block.Hash()
|
||||
blockHashStr := blockHash.String()
|
||||
height := block.NumberU64()
|
||||
traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
|
||||
transactions := block.Transactions()
|
||||
// Derive any missing fields
|
||||
if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Generate the block iplds
|
||||
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
|
||||
}
|
||||
|
||||
if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) {
|
||||
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
|
||||
}
|
||||
if len(txTrieNodes) != len(rctTrieNodes) {
|
||||
return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
|
||||
}
|
||||
|
||||
// Calculate reward
|
||||
var reward *big.Int
|
||||
// in PoA networks block reward is 0
|
||||
if sdi.chainConfig.Clique != nil {
|
||||
reward = big.NewInt(0)
|
||||
} else {
|
||||
reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts)
|
||||
}
|
||||
t = time.Now()
|
||||
|
||||
blockTx := &BatchTx{
|
||||
BlockNumber: block.Number().String(),
|
||||
submit: func(self *BatchTx, err error) error {
|
||||
tDiff := time.Since(t)
|
||||
metrics.IndexerMetrics.StateStoreCodeProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
sdi.fileWriter.Flush()
|
||||
tDiff = time.Since(t)
|
||||
metrics.IndexerMetrics.PostgresCommitTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
|
||||
traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
|
||||
log.Debug(traceMsg)
|
||||
return err
|
||||
},
|
||||
}
|
||||
tDiff := time.Since(t)
|
||||
metrics.IndexerMetrics.FreePostgresTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
|
||||
// write header, collect headerID
|
||||
headerID := sdi.processHeader(block.Header(), headerNode, reward, totalDifficulty)
|
||||
tDiff = time.Since(t)
|
||||
metrics.IndexerMetrics.HeaderProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
|
||||
// write uncles
|
||||
sdi.processUncles(headerID, block.Number(), uncleNodes)
|
||||
tDiff = time.Since(t)
|
||||
metrics.IndexerMetrics.UncleProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
|
||||
// write receipts and txs
|
||||
err = sdi.processReceiptsAndTxs(processArgs{
|
||||
headerID: headerID,
|
||||
blockNumber: block.Number(),
|
||||
receipts: receipts,
|
||||
txs: transactions,
|
||||
rctNodes: rctNodes,
|
||||
rctTrieNodes: rctTrieNodes,
|
||||
txNodes: txNodes,
|
||||
txTrieNodes: txTrieNodes,
|
||||
logTrieNodes: logTrieNodes,
|
||||
logLeafNodeCIDs: logLeafNodeCIDs,
|
||||
rctLeafNodeCIDs: rctLeafNodeCIDs,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tDiff = time.Since(t)
|
||||
metrics.IndexerMetrics.TxAndRecProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
|
||||
return blockTx, err
|
||||
}
|
||||
|
||||
// processHeader write a header IPLD insert SQL stmt to a file
|
||||
// it returns the headerID
|
||||
func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode node.Node, reward, td *big.Int) string {
|
||||
sdi.fileWriter.upsertIPLDNode(header.Number.String(), headerNode)
|
||||
|
||||
var baseFee *string
|
||||
if header.BaseFee != nil {
|
||||
baseFee = new(string)
|
||||
*baseFee = header.BaseFee.String()
|
||||
}
|
||||
headerID := header.Hash().String()
|
||||
sdi.fileWriter.upsertHeaderCID(models.HeaderModel{
|
||||
NodeID: sdi.nodeID,
|
||||
CID: headerNode.Cid().String(),
|
||||
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
|
||||
ParentHash: header.ParentHash.String(),
|
||||
BlockNumber: header.Number.String(),
|
||||
BlockHash: headerID,
|
||||
TotalDifficulty: td.String(),
|
||||
Reward: reward.String(),
|
||||
Bloom: header.Bloom.Bytes(),
|
||||
StateRoot: header.Root.String(),
|
||||
RctRoot: header.ReceiptHash.String(),
|
||||
TxRoot: header.TxHash.String(),
|
||||
UncleRoot: header.UncleHash.String(),
|
||||
Timestamp: header.Time,
|
||||
Coinbase: header.Coinbase.String(),
|
||||
})
|
||||
return headerID
|
||||
}
|
||||
|
||||
// processUncles writes uncle IPLD insert SQL stmts to a file
|
||||
func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber *big.Int, uncleNodes []*ipld2.EthHeader) {
|
||||
// publish and index uncles
|
||||
for _, uncleNode := range uncleNodes {
|
||||
sdi.fileWriter.upsertIPLDNode(blockNumber.String(), uncleNode)
|
||||
var uncleReward *big.Int
|
||||
// in PoA networks uncle reward is 0
|
||||
if sdi.chainConfig.Clique != nil {
|
||||
uncleReward = big.NewInt(0)
|
||||
} else {
|
||||
uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncleNode.Number.Uint64())
|
||||
}
|
||||
sdi.fileWriter.upsertUncleCID(models.UncleModel{
|
||||
BlockNumber: blockNumber.String(),
|
||||
HeaderID: headerID,
|
||||
CID: uncleNode.Cid().String(),
|
||||
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
|
||||
ParentHash: uncleNode.ParentHash.String(),
|
||||
BlockHash: uncleNode.Hash().String(),
|
||||
Reward: uncleReward.String(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// processArgs bundles arguments to processReceiptsAndTxs
|
||||
type processArgs struct {
|
||||
headerID string
|
||||
blockNumber *big.Int
|
||||
receipts types.Receipts
|
||||
txs types.Transactions
|
||||
rctNodes []*ipld2.EthReceipt
|
||||
rctTrieNodes []*ipld2.EthRctTrie
|
||||
txNodes []*ipld2.EthTx
|
||||
txTrieNodes []*ipld2.EthTxTrie
|
||||
logTrieNodes [][]node.Node
|
||||
logLeafNodeCIDs [][]cid.Cid
|
||||
rctLeafNodeCIDs []cid.Cid
|
||||
}
|
||||
|
||||
// processReceiptsAndTxs writes receipt and tx IPLD insert SQL stmts to a file
|
||||
func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
|
||||
// Process receipts and txs
|
||||
signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
|
||||
for i, receipt := range args.receipts {
|
||||
for _, logTrieNode := range args.logTrieNodes[i] {
|
||||
sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), logTrieNode)
|
||||
}
|
||||
txNode := args.txNodes[i]
|
||||
sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), txNode)
|
||||
|
||||
// index tx
|
||||
trx := args.txs[i]
|
||||
txID := trx.Hash().String()
|
||||
|
||||
var val string
|
||||
if trx.Value() != nil {
|
||||
val = trx.Value().String()
|
||||
}
|
||||
|
||||
// derive sender for the tx that corresponds with this receipt
|
||||
from, err := types.Sender(signer, trx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deriving tx sender: %v", err)
|
||||
}
|
||||
txModel := models.TxModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
HeaderID: args.headerID,
|
||||
Dst: shared.HandleZeroAddrPointer(trx.To()),
|
||||
Src: shared.HandleZeroAddr(from),
|
||||
TxHash: txID,
|
||||
Index: int64(i),
|
||||
Data: trx.Data(),
|
||||
CID: txNode.Cid().String(),
|
||||
MhKey: shared.MultihashKeyFromCID(txNode.Cid()),
|
||||
Type: trx.Type(),
|
||||
Value: val,
|
||||
}
|
||||
sdi.fileWriter.upsertTransactionCID(txModel)
|
||||
|
||||
// index access list if this is one
|
||||
for j, accessListElement := range trx.AccessList() {
|
||||
storageKeys := make([]string, len(accessListElement.StorageKeys))
|
||||
for k, storageKey := range accessListElement.StorageKeys {
|
||||
storageKeys[k] = storageKey.Hex()
|
||||
}
|
||||
accessListElementModel := models.AccessListElementModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
TxID: txID,
|
||||
Index: int64(j),
|
||||
Address: accessListElement.Address.Hex(),
|
||||
StorageKeys: storageKeys,
|
||||
}
|
||||
sdi.fileWriter.upsertAccessListElement(accessListElementModel)
|
||||
}
|
||||
|
||||
// this is the contract address if this receipt is for a contract creation tx
|
||||
contract := shared.HandleZeroAddr(receipt.ContractAddress)
|
||||
var contractHash string
|
||||
if contract != "" {
|
||||
contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
|
||||
}
|
||||
|
||||
// index receipt
|
||||
if !args.rctLeafNodeCIDs[i].Defined() {
|
||||
return fmt.Errorf("invalid receipt leaf node cid")
|
||||
}
|
||||
|
||||
rctModel := &models.ReceiptModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
HeaderID: args.headerID,
|
||||
TxID: txID,
|
||||
Contract: contract,
|
||||
ContractHash: contractHash,
|
||||
LeafCID: args.rctLeafNodeCIDs[i].String(),
|
||||
LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]),
|
||||
LogRoot: args.rctNodes[i].LogRoot.String(),
|
||||
}
|
||||
if len(receipt.PostState) == 0 {
|
||||
rctModel.PostStatus = receipt.Status
|
||||
} else {
|
||||
rctModel.PostState = common.Bytes2Hex(receipt.PostState)
|
||||
}
|
||||
sdi.fileWriter.upsertReceiptCID(rctModel)
|
||||
|
||||
// index logs
|
||||
logDataSet := make([]*models.LogsModel, len(receipt.Logs))
|
||||
for idx, l := range receipt.Logs {
|
||||
topicSet := make([]string, 4)
|
||||
for ti, topic := range l.Topics {
|
||||
topicSet[ti] = topic.Hex()
|
||||
}
|
||||
|
||||
if !args.logLeafNodeCIDs[i][idx].Defined() {
|
||||
return fmt.Errorf("invalid log cid")
|
||||
}
|
||||
|
||||
logDataSet[idx] = &models.LogsModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
HeaderID: args.headerID,
|
||||
ReceiptID: txID,
|
||||
Address: l.Address.String(),
|
||||
Index: int64(l.Index),
|
||||
Data: l.Data,
|
||||
LeafCID: args.logLeafNodeCIDs[i][idx].String(),
|
||||
LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]),
|
||||
Topic0: topicSet[0],
|
||||
Topic1: topicSet[1],
|
||||
Topic2: topicSet[2],
|
||||
Topic3: topicSet[3],
|
||||
}
|
||||
}
|
||||
sdi.fileWriter.upsertLogCID(logDataSet)
|
||||
}
|
||||
|
||||
// publish trie nodes, these aren't indexed directly
|
||||
for i, n := range args.txTrieNodes {
|
||||
sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), n)
|
||||
sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), args.rctTrieNodes[i])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PushStateNode writes a state diff node object (including any child storage nodes) IPLD insert SQL stmt to a file
|
||||
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error {
|
||||
tx, ok := batch.(*BatchTx)
|
||||
if !ok {
|
||||
return fmt.Errorf("file: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
|
||||
}
|
||||
// publish the state node
|
||||
var stateModel models.StateNodeModel
|
||||
if stateNode.NodeType == sdtypes.Removed {
|
||||
if atomic.LoadUint32(sdi.removedCacheFlag) == 0 {
|
||||
atomic.StoreUint32(sdi.removedCacheFlag, 1)
|
||||
sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, shared.RemovedNodeMhKey, []byte{})
|
||||
}
|
||||
stateModel = models.StateNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
Path: stateNode.Path,
|
||||
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
|
||||
CID: shared.RemovedNodeStateCID,
|
||||
MhKey: shared.RemovedNodeMhKey,
|
||||
NodeType: stateNode.NodeType.Int(),
|
||||
}
|
||||
} else {
|
||||
stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(tx.BlockNumber, ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error generating and cacheing state node IPLD: %v", err)
|
||||
}
|
||||
stateModel = models.StateNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
Path: stateNode.Path,
|
||||
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
|
||||
CID: stateCIDStr,
|
||||
MhKey: stateMhKey,
|
||||
NodeType: stateNode.NodeType.Int(),
|
||||
}
|
||||
}
|
||||
|
||||
// index the state node
|
||||
sdi.fileWriter.upsertStateCID(stateModel)
|
||||
|
||||
// if we have a leaf, decode and index the account data
|
||||
if stateNode.NodeType == sdtypes.Leaf {
|
||||
var i []interface{}
|
||||
if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil {
|
||||
return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
|
||||
}
|
||||
if len(i) != 2 {
|
||||
return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements")
|
||||
}
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
|
||||
return fmt.Errorf("error decoding state account rlp: %s", err.Error())
|
||||
}
|
||||
accountModel := models.StateAccountModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
StatePath: stateNode.Path,
|
||||
Balance: account.Balance.String(),
|
||||
Nonce: account.Nonce,
|
||||
CodeHash: account.CodeHash,
|
||||
StorageRoot: account.Root.String(),
|
||||
}
|
||||
sdi.fileWriter.upsertStateAccount(accountModel)
|
||||
}
|
||||
|
||||
// if there are any storage nodes associated with this node, publish and index them
|
||||
for _, storageNode := range stateNode.StorageNodes {
|
||||
if storageNode.NodeType == sdtypes.Removed {
|
||||
if atomic.LoadUint32(sdi.removedCacheFlag) == 0 {
|
||||
atomic.StoreUint32(sdi.removedCacheFlag, 1)
|
||||
sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, shared.RemovedNodeMhKey, []byte{})
|
||||
}
|
||||
storageModel := models.StorageNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
StatePath: stateNode.Path,
|
||||
Path: storageNode.Path,
|
||||
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
|
||||
CID: shared.RemovedNodeStorageCID,
|
||||
MhKey: shared.RemovedNodeMhKey,
|
||||
NodeType: storageNode.NodeType.Int(),
|
||||
}
|
||||
sdi.fileWriter.upsertStorageCID(storageModel)
|
||||
continue
|
||||
}
|
||||
storageCIDStr, storageMhKey, err := sdi.fileWriter.upsertIPLDRaw(tx.BlockNumber, ipld2.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err)
|
||||
}
|
||||
storageModel := models.StorageNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
StatePath: stateNode.Path,
|
||||
Path: storageNode.Path,
|
||||
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
|
||||
CID: storageCIDStr,
|
||||
MhKey: storageMhKey,
|
||||
NodeType: storageNode.NodeType.Int(),
|
||||
}
|
||||
sdi.fileWriter.upsertStorageCID(storageModel)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PushCodeAndCodeHash writes code and codehash pairs insert SQL stmts to a file
|
||||
func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error {
|
||||
tx, ok := batch.(*BatchTx)
|
||||
if !ok {
|
||||
return fmt.Errorf("file: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
|
||||
}
|
||||
// codec doesn't matter since db key is multihash-based
|
||||
mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deriving multihash key from codehash: %v", err)
|
||||
}
|
||||
sdi.fileWriter.upsertIPLDDirect(tx.BlockNumber, mhKey, codeAndCodeHash.Code)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close satisfies io.Closer
|
||||
func (sdi *StateDiffIndexer) Close() error {
|
||||
return sdi.fileWriter.Close()
|
||||
}
|
||||
|
||||
// LoadWatchedAddresses loads watched addresses from a file
|
||||
func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
|
||||
return sdi.fileWriter.loadWatchedAddresses()
|
||||
}
|
||||
|
||||
// InsertWatchedAddresses inserts the given addresses in a file
|
||||
func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
|
||||
return sdi.fileWriter.insertWatchedAddresses(args, currentBlockNumber)
|
||||
}
|
||||
|
||||
// RemoveWatchedAddresses removes the given watched addresses from a file
|
||||
func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) error {
|
||||
return sdi.fileWriter.removeWatchedAddresses(args)
|
||||
}
|
||||
|
||||
// SetWatchedAddresses clears and inserts the given addresses in a file
|
||||
func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error {
|
||||
return sdi.fileWriter.setWatchedAddresses(args, currentBlockNumber)
|
||||
}
|
||||
|
||||
// ClearWatchedAddresses clears all the watched addresses from a file
|
||||
func (sdi *StateDiffIndexer) ClearWatchedAddresses() error {
|
||||
return sdi.SetWatchedAddresses([]sdtypes.WatchAddressArg{}, big.NewInt(0))
|
||||
}
|
||||
60
statediff/indexer/database/file/interfaces.go
Normal file
60
statediff/indexer/database/file/interfaces.go
Normal file
@ -0,0 +1,60 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2022 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/models"
|
||||
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
"github.com/ethereum/go-ethereum/statediff/types"
|
||||
)
|
||||
|
||||
// Writer interface required by the file indexer
|
||||
type FileWriter interface {
|
||||
// Methods used to control the writer
|
||||
Loop()
|
||||
Close() error
|
||||
Flush()
|
||||
|
||||
// Methods to upsert ethereum data model objects
|
||||
upsertNode(node nodeinfo.Info)
|
||||
upsertHeaderCID(header models.HeaderModel)
|
||||
upsertUncleCID(uncle models.UncleModel)
|
||||
upsertTransactionCID(transaction models.TxModel)
|
||||
upsertAccessListElement(accessListElement models.AccessListElementModel)
|
||||
upsertReceiptCID(rct *models.ReceiptModel)
|
||||
upsertLogCID(logs []*models.LogsModel)
|
||||
upsertStateCID(stateNode models.StateNodeModel)
|
||||
upsertStateAccount(stateAccount models.StateAccountModel)
|
||||
upsertStorageCID(storageCID models.StorageNodeModel)
|
||||
upsertIPLD(ipld models.IPLDModel)
|
||||
|
||||
// Methods to upsert IPLD in different ways
|
||||
upsertIPLDDirect(blockNumber, key string, value []byte)
|
||||
upsertIPLDNode(blockNumber string, i node.Node)
|
||||
upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error)
|
||||
|
||||
// Methods to read and write watched addresses
|
||||
loadWatchedAddresses() ([]common.Address, error)
|
||||
insertWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error
|
||||
removeWatchedAddresses(args []types.WatchAddressArg) error
|
||||
setWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error
|
||||
}
|
||||
112
statediff/indexer/database/file/mainnet_tests/indexer_test.go
Normal file
112
statediff/indexer/database/file/mainnet_tests/indexer_test.go
Normal file
@ -0,0 +1,112 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package mainnet_tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
|
||||
)
|
||||
|
||||
var (
|
||||
err error
|
||||
db sql.Database
|
||||
ind interfaces.StateDiffIndexer
|
||||
chainConf = params.MainnetChainConfig
|
||||
)
|
||||
|
||||
func init() {
|
||||
if os.Getenv("MODE") != "statediff" {
|
||||
fmt.Println("Skipping statediff test")
|
||||
os.Exit(0)
|
||||
}
|
||||
if os.Getenv("STATEDIFF_DB") != "file" {
|
||||
fmt.Println("Skipping statediff .sql file writing mode test")
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPushBlockAndState(t *testing.T) {
|
||||
conf := test_helpers.GetTestConfig()
|
||||
|
||||
for _, blockNumber := range test_helpers.ProblemBlocks {
|
||||
conf.BlockNumber = big.NewInt(blockNumber)
|
||||
tb, trs, err := test_helpers.TestBlockAndReceipts(conf)
|
||||
require.NoError(t, err)
|
||||
|
||||
testPushBlockAndState(t, tb, trs)
|
||||
}
|
||||
|
||||
testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf)
|
||||
require.NoError(t, err)
|
||||
|
||||
testPushBlockAndState(t, testBlock, testReceipts)
|
||||
}
|
||||
|
||||
func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
|
||||
t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
|
||||
setupMainnetIndexer(t)
|
||||
defer dumpData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestBlock(t, ind, block, receipts)
|
||||
})
|
||||
}
|
||||
|
||||
func setupMainnetIndexer(t *testing.T) {
|
||||
if _, err := os.Stat(file.CSVTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
|
||||
err := os.Remove(file.CSVTestConfig.FilePath)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
ind, err = file.NewStateDiffIndexer(context.Background(), chainConf, file.CSVTestConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err = postgres.SetupSQLXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func dumpData(t *testing.T) {
|
||||
sqlFileBytes, err := os.ReadFile(file.CSVTestConfig.FilePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Exec(context.Background(), string(sqlFileBytes))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func tearDown(t *testing.T) {
|
||||
test_helpers.TearDownDB(t, db)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
require.NoError(t, os.Remove(file.CSVTestConfig.FilePath))
|
||||
}
|
||||
101
statediff/indexer/database/file/sql_indexer_legacy_test.go
Normal file
101
statediff/indexer/database/file/sql_indexer_legacy_test.go
Normal file
@ -0,0 +1,101 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
|
||||
)
|
||||
|
||||
var (
|
||||
db sql.Database
|
||||
err error
|
||||
ind interfaces.StateDiffIndexer
|
||||
)
|
||||
|
||||
func setupLegacySQLIndexer(t *testing.T) {
|
||||
if _, err := os.Stat(file.SQLTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
|
||||
err := os.Remove(file.SQLTestConfig.FilePath)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.SQLTestConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err = postgres.SetupSQLXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func setupLegacySQL(t *testing.T) {
|
||||
setupLegacySQLIndexer(t)
|
||||
test.SetupLegacyTestData(t, ind)
|
||||
}
|
||||
|
||||
func dumpFileData(t *testing.T) {
|
||||
err := test_helpers.DedupFile(file.SQLTestConfig.FilePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
sqlFileBytes, err := os.ReadFile(file.SQLTestConfig.FilePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Exec(context.Background(), string(sqlFileBytes))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func resetAndDumpWatchedAddressesFileData(t *testing.T) {
|
||||
test_helpers.TearDownDB(t, db)
|
||||
|
||||
sqlFileBytes, err := os.ReadFile(file.SQLTestConfig.WatchedAddressesFilePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Exec(context.Background(), string(sqlFileBytes))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func tearDown(t *testing.T) {
|
||||
test_helpers.TearDownDB(t, db)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
require.NoError(t, os.Remove(file.SQLTestConfig.FilePath))
|
||||
|
||||
if err := os.Remove(file.SQLTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacySQLFileIndexer(t *testing.T) {
|
||||
t.Run("Publish and index header IPLDs", func(t *testing.T) {
|
||||
setupLegacySQL(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestLegacyIndexer(t, db)
|
||||
})
|
||||
}
|
||||
253
statediff/indexer/database/file/sql_indexer_test.go
Normal file
253
statediff/indexer/database/file/sql_indexer_test.go
Normal file
@ -0,0 +1,253 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math/big"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
)
|
||||
|
||||
func setupIndexer(t *testing.T) {
|
||||
if _, err := os.Stat(file.SQLTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
|
||||
err := os.Remove(file.SQLTestConfig.FilePath)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(file.SQLTestConfig.WatchedAddressesFilePath); !errors.Is(err, os.ErrNotExist) {
|
||||
err := os.Remove(file.SQLTestConfig.WatchedAddressesFilePath)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.SQLTestConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err = postgres.SetupSQLXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func setup(t *testing.T) {
|
||||
setupIndexer(t)
|
||||
test.SetupTestData(t, ind)
|
||||
}
|
||||
|
||||
func setupSQLNonCanonical(t *testing.T) {
|
||||
setupIndexer(t)
|
||||
test.SetupTestDataNonCanonical(t, ind)
|
||||
}
|
||||
|
||||
func TestSQLFileIndexer(t *testing.T) {
|
||||
t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
|
||||
setup(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexHeaderIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
|
||||
setup(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexTransactionIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
|
||||
setup(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexLogIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
|
||||
setup(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexReceiptIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
|
||||
setup(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexStateIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
|
||||
setup(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexStorageIPLDs(t, db)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSQLFileIndexerNonCanonical(t *testing.T) {
|
||||
t.Run("Publish and index header", func(t *testing.T) {
|
||||
setupSQLNonCanonical(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexHeaderNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index transactions", func(t *testing.T) {
|
||||
setupSQLNonCanonical(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index receipts", func(t *testing.T) {
|
||||
setupSQLNonCanonical(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index logs", func(t *testing.T) {
|
||||
setupSQLNonCanonical(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexLogsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index state nodes", func(t *testing.T) {
|
||||
setupSQLNonCanonical(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexStateNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index storage nodes", func(t *testing.T) {
|
||||
setupSQLNonCanonical(t)
|
||||
dumpFileData(t)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestPublishAndIndexStorageNonCanonical(t, db)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSQLFileWatchAddressMethods(t *testing.T) {
|
||||
setupIndexer(t)
|
||||
defer tearDown(t)
|
||||
|
||||
t.Run("Load watched addresses (empty table)", func(t *testing.T) {
|
||||
test.TestLoadEmptyWatchedAddresses(t, ind)
|
||||
})
|
||||
|
||||
t.Run("Insert watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetInsertWatchedAddressesArgs()
|
||||
err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesFileData(t)
|
||||
|
||||
test.TestInsertWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
|
||||
args := mocks.GetInsertAlreadyWatchedAddressesArgs()
|
||||
err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesFileData(t)
|
||||
|
||||
test.TestInsertAlreadyWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Remove watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetRemoveWatchedAddressesArgs()
|
||||
err = ind.RemoveWatchedAddresses(args)
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesFileData(t)
|
||||
|
||||
test.TestRemoveWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
|
||||
args := mocks.GetRemoveNonWatchedAddressesArgs()
|
||||
err = ind.RemoveWatchedAddresses(args)
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesFileData(t)
|
||||
|
||||
test.TestRemoveNonWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Set watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetSetWatchedAddressesArgs()
|
||||
err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesFileData(t)
|
||||
|
||||
test.TestSetWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
|
||||
args := mocks.GetSetAlreadyWatchedAddressesArgs()
|
||||
err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesFileData(t)
|
||||
|
||||
test.TestSetAlreadyWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Load watched addresses", func(t *testing.T) {
|
||||
test.TestLoadWatchedAddresses(t, ind)
|
||||
})
|
||||
|
||||
t.Run("Clear watched addresses", func(t *testing.T) {
|
||||
err = ind.ClearWatchedAddresses()
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesFileData(t)
|
||||
|
||||
test.TestClearWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
|
||||
err = ind.ClearWatchedAddresses()
|
||||
require.NoError(t, err)
|
||||
|
||||
resetAndDumpWatchedAddressesFileData(t)
|
||||
|
||||
test.TestClearEmptyWatchedAddresses(t, db)
|
||||
})
|
||||
}
|
||||
431
statediff/indexer/database/file/sql_writer.go
Normal file
431
statediff/indexer/database/file/sql_writer.go
Normal file
@ -0,0 +1,431 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"os"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
dshelp "github.com/ipfs/go-ipfs-ds-help"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
pg_query "github.com/pganalyze/pg_query_go/v2"
|
||||
"github.com/thoas/go-funk"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/models"
|
||||
nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
"github.com/ethereum/go-ethereum/statediff/types"
|
||||
)
|
||||
|
||||
var (
|
||||
nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
|
||||
pipeSize = 65336 // min(linuxPipeSize, macOSPipeSize)
|
||||
writeBufferSize = pipeSize * 16 * 96
|
||||
)
|
||||
|
||||
// SQLWriter writes sql statements to a file
|
||||
type SQLWriter struct {
|
||||
wc io.WriteCloser
|
||||
stmts chan []byte
|
||||
collatedStmt []byte
|
||||
collationIndex int
|
||||
|
||||
flushChan chan struct{}
|
||||
flushFinished chan struct{}
|
||||
quitChan chan struct{}
|
||||
doneChan chan struct{}
|
||||
|
||||
watchedAddressesFilePath string
|
||||
}
|
||||
|
||||
// NewSQLWriter creates a new pointer to a Writer
|
||||
func NewSQLWriter(wc io.WriteCloser, watchedAddressesFilePath string) *SQLWriter {
|
||||
return &SQLWriter{
|
||||
wc: wc,
|
||||
stmts: make(chan []byte),
|
||||
collatedStmt: make([]byte, writeBufferSize),
|
||||
flushChan: make(chan struct{}),
|
||||
flushFinished: make(chan struct{}),
|
||||
quitChan: make(chan struct{}),
|
||||
doneChan: make(chan struct{}),
|
||||
watchedAddressesFilePath: watchedAddressesFilePath,
|
||||
}
|
||||
}
|
||||
|
||||
// Loop enables concurrent writes to the underlying os.File
|
||||
// since os.File does not buffer, it utilizes an internal buffer that is the size of a unix pipe
|
||||
// by using copy() and tracking the index/size of the buffer, we require only the initial memory allocation
|
||||
func (sqw *SQLWriter) Loop() {
|
||||
sqw.collationIndex = 0
|
||||
go func() {
|
||||
defer close(sqw.doneChan)
|
||||
var l int
|
||||
for {
|
||||
select {
|
||||
case stmt := <-sqw.stmts:
|
||||
l = len(stmt)
|
||||
if sqw.collationIndex+l > writeBufferSize {
|
||||
if err := sqw.flush(); err != nil {
|
||||
panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err))
|
||||
}
|
||||
if l > writeBufferSize {
|
||||
if _, err := sqw.wc.Write(stmt); err != nil {
|
||||
panic(fmt.Sprintf("error writing large sql stmt to file: %v", err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
copy(sqw.collatedStmt[sqw.collationIndex:sqw.collationIndex+l], stmt)
|
||||
sqw.collationIndex += l
|
||||
case <-sqw.quitChan:
|
||||
if err := sqw.flush(); err != nil {
|
||||
panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err))
|
||||
}
|
||||
return
|
||||
case <-sqw.flushChan:
|
||||
if err := sqw.flush(); err != nil {
|
||||
panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err))
|
||||
}
|
||||
sqw.flushFinished <- struct{}{}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Close satisfies io.Closer
|
||||
func (sqw *SQLWriter) Close() error {
|
||||
close(sqw.quitChan)
|
||||
<-sqw.doneChan
|
||||
close(sqw.stmts)
|
||||
close(sqw.flushChan)
|
||||
close(sqw.flushFinished)
|
||||
return sqw.wc.Close()
|
||||
}
|
||||
|
||||
// Flush sends a flush signal to the looping process
|
||||
func (sqw *SQLWriter) Flush() {
|
||||
sqw.flushChan <- struct{}{}
|
||||
<-sqw.flushFinished
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) flush() error {
|
||||
if _, err := sqw.wc.Write(sqw.collatedStmt[0:sqw.collationIndex]); err != nil {
|
||||
return err
|
||||
}
|
||||
sqw.collationIndex = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
nodeInsert = "INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES " +
|
||||
"('%s', '%s', '%s', '%s', %d);\n"
|
||||
|
||||
ipldInsert = "INSERT INTO public.blocks (block_number, key, data) VALUES ('%s', '%s', '\\x%x');\n"
|
||||
|
||||
headerInsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, " +
|
||||
"state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES " +
|
||||
"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s', %d, '%s');\n"
|
||||
|
||||
uncleInsert = "INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES " +
|
||||
"('%s', '%s', '%s', '%s', '%s', '%s', '%s');\n"
|
||||
|
||||
txInsert = "INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, " +
|
||||
"value) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '\\x%x', %d, '%s');\n"
|
||||
|
||||
alInsert = "INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES " +
|
||||
"('%s', '%s', %d, '%s', '%s');\n"
|
||||
|
||||
rctInsert = "INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, " +
|
||||
"post_status, log_root) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s');\n"
|
||||
|
||||
logInsert = "INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, " +
|
||||
"topic3, log_data) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '\\x%x');\n"
|
||||
|
||||
stateInsert = "INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) " +
|
||||
"VALUES ('%s', '%s', '%s', '%s', '\\x%x', %d, %t, '%s');\n"
|
||||
|
||||
accountInsert = "INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) " +
|
||||
"VALUES ('%s', '%s', '\\x%x', '%s', %d, '\\x%x', '%s');\n"
|
||||
|
||||
storageInsert = "INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, " +
|
||||
"node_type, diff, mh_key) VALUES ('%s', '%s', '\\x%x', '%s', '%s', '\\x%x', %d, %t, '%s');\n"
|
||||
)
|
||||
|
||||
func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) {
|
||||
sqw.stmts <- []byte(fmt.Sprintf(nodeInsert, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID))
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertIPLD(ipld models.IPLDModel) {
|
||||
sqw.stmts <- []byte(fmt.Sprintf(ipldInsert, ipld.BlockNumber, ipld.Key, ipld.Data))
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertIPLDDirect(blockNumber, key string, value []byte) {
|
||||
sqw.upsertIPLD(models.IPLDModel{
|
||||
BlockNumber: blockNumber,
|
||||
Key: key,
|
||||
Data: value,
|
||||
})
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertIPLDNode(blockNumber string, i node.Node) {
|
||||
sqw.upsertIPLD(models.IPLDModel{
|
||||
BlockNumber: blockNumber,
|
||||
Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(),
|
||||
Data: i.RawData(),
|
||||
})
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error) {
|
||||
c, err := ipld.RawdataToCid(codec, raw, mh)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
|
||||
sqw.upsertIPLD(models.IPLDModel{
|
||||
BlockNumber: blockNumber,
|
||||
Key: prefixedKey,
|
||||
Data: raw,
|
||||
})
|
||||
return c.String(), prefixedKey, err
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) {
|
||||
stmt := fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
|
||||
header.TotalDifficulty, header.NodeID, header.Reward, header.StateRoot, header.TxRoot,
|
||||
header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1, header.Coinbase)
|
||||
sqw.stmts <- []byte(stmt)
|
||||
metrics.IndexerMetrics.BlocksCounter.Inc(1)
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertUncleCID(uncle models.UncleModel) {
|
||||
sqw.stmts <- []byte(fmt.Sprintf(uncleInsert, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID,
|
||||
uncle.Reward, uncle.MhKey))
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertTransactionCID(transaction models.TxModel) {
|
||||
sqw.stmts <- []byte(fmt.Sprintf(txInsert, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
|
||||
transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value))
|
||||
metrics.IndexerMetrics.TransactionsCounter.Inc(1)
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertAccessListElement(accessListElement models.AccessListElementModel) {
|
||||
sqw.stmts <- []byte(fmt.Sprintf(alInsert, accessListElement.BlockNumber, accessListElement.TxID, accessListElement.Index, accessListElement.Address,
|
||||
formatPostgresStringArray(accessListElement.StorageKeys)))
|
||||
metrics.IndexerMetrics.AccessListEntriesCounter.Inc(1)
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) {
|
||||
sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey,
|
||||
rct.PostState, rct.PostStatus, rct.LogRoot))
|
||||
metrics.IndexerMetrics.ReceiptsCounter.Inc(1)
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) {
|
||||
for _, l := range logs {
|
||||
sqw.stmts <- []byte(fmt.Sprintf(logInsert, l.BlockNumber, l.HeaderID, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0,
|
||||
l.Topic1, l.Topic2, l.Topic3, l.Data))
|
||||
metrics.IndexerMetrics.LogsCounter.Inc(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertStateCID(stateNode models.StateNodeModel) {
|
||||
var stateKey string
|
||||
if stateNode.StateKey != nullHash.String() {
|
||||
stateKey = stateNode.StateKey
|
||||
}
|
||||
sqw.stmts <- []byte(fmt.Sprintf(stateInsert, stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path,
|
||||
stateNode.NodeType, true, stateNode.MhKey))
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertStateAccount(stateAccount models.StateAccountModel) {
|
||||
sqw.stmts <- []byte(fmt.Sprintf(accountInsert, stateAccount.BlockNumber, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance,
|
||||
stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot))
|
||||
}
|
||||
|
||||
func (sqw *SQLWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
|
||||
var storageKey string
|
||||
if storageCID.StorageKey != nullHash.String() {
|
||||
storageKey = storageCID.StorageKey
|
||||
}
|
||||
sqw.stmts <- []byte(fmt.Sprintf(storageInsert, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID,
|
||||
storageCID.Path, storageCID.NodeType, true, storageCID.MhKey))
|
||||
}
|
||||
|
||||
// LoadWatchedAddresses loads watched addresses from a file
|
||||
func (sqw *SQLWriter) loadWatchedAddresses() ([]common.Address, error) {
|
||||
// load sql statements from watched addresses file
|
||||
stmts, err := loadWatchedAddressesStatements(sqw.watchedAddressesFilePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// extract addresses from the sql statements
|
||||
watchedAddresses := []common.Address{}
|
||||
for _, stmt := range stmts {
|
||||
addressString, err := parseWatchedAddressStatement(stmt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString))
|
||||
}
|
||||
|
||||
return watchedAddresses, nil
|
||||
}
|
||||
|
||||
// InsertWatchedAddresses inserts the given addresses in a file
|
||||
func (sqw *SQLWriter) insertWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error {
|
||||
// load sql statements from watched addresses file
|
||||
stmts, err := loadWatchedAddressesStatements(sqw.watchedAddressesFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// get already watched addresses
|
||||
var watchedAddresses []string
|
||||
for _, stmt := range stmts {
|
||||
addressString, err := parseWatchedAddressStatement(stmt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
watchedAddresses = append(watchedAddresses, addressString)
|
||||
}
|
||||
|
||||
// append statements for new addresses to existing statements
|
||||
for _, arg := range args {
|
||||
// ignore if already watched
|
||||
if funk.Contains(watchedAddresses, arg.Address) {
|
||||
continue
|
||||
}
|
||||
|
||||
stmt := fmt.Sprintf(watchedAddressesInsert, arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
|
||||
stmts = append(stmts, stmt)
|
||||
}
|
||||
|
||||
return dumpWatchedAddressesStatements(sqw.watchedAddressesFilePath, stmts)
|
||||
}
|
||||
|
||||
// RemoveWatchedAddresses removes the given watched addresses from a file
|
||||
func (sqw *SQLWriter) removeWatchedAddresses(args []types.WatchAddressArg) error {
|
||||
// load sql statements from watched addresses file
|
||||
stmts, err := loadWatchedAddressesStatements(sqw.watchedAddressesFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// get rid of statements having addresses to be removed
|
||||
var filteredStmts []string
|
||||
for _, stmt := range stmts {
|
||||
addressString, err := parseWatchedAddressStatement(stmt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
toRemove := funk.Contains(args, func(arg types.WatchAddressArg) bool {
|
||||
return arg.Address == addressString
|
||||
})
|
||||
|
||||
if !toRemove {
|
||||
filteredStmts = append(filteredStmts, stmt)
|
||||
}
|
||||
}
|
||||
|
||||
return dumpWatchedAddressesStatements(sqw.watchedAddressesFilePath, filteredStmts)
|
||||
}
|
||||
|
||||
// SetWatchedAddresses clears and inserts the given addresses in a file
|
||||
func (sqw *SQLWriter) setWatchedAddresses(args []types.WatchAddressArg, currentBlockNumber *big.Int) error {
|
||||
var stmts []string
|
||||
for _, arg := range args {
|
||||
stmt := fmt.Sprintf(watchedAddressesInsert, arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
|
||||
stmts = append(stmts, stmt)
|
||||
}
|
||||
|
||||
return dumpWatchedAddressesStatements(sqw.watchedAddressesFilePath, stmts)
|
||||
}
|
||||
|
||||
// loadWatchedAddressesStatements loads sql statements from the given file in a string slice
|
||||
func loadWatchedAddressesStatements(filePath string) ([]string, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("error opening watched addresses file: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stmts := []string{}
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
stmts = append(stmts, scanner.Text())
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("error loading watched addresses: %v", err)
|
||||
}
|
||||
|
||||
return stmts, nil
|
||||
}
|
||||
|
||||
// dumpWatchedAddressesStatements dumps sql statements to the given file
|
||||
func dumpWatchedAddressesStatements(filePath string, stmts []string) error {
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating watched addresses file: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
for _, stmt := range stmts {
|
||||
_, err := file.Write([]byte(stmt + "\n"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error inserting watched_addresses entry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseWatchedAddressStatement parses given sql insert statement to extract the address argument
|
||||
func parseWatchedAddressStatement(stmt string) (string, error) {
|
||||
parseResult, err := pg_query.Parse(stmt)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error parsing sql stmt: %v", err)
|
||||
}
|
||||
|
||||
// extract address argument from parse output for a SQL statement of form
|
||||
// "INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at)
|
||||
// VALUES ('0xabc', '123', '130') ON CONFLICT (address) DO NOTHING;"
|
||||
addressString := parseResult.Stmts[0].Stmt.GetInsertStmt().
|
||||
SelectStmt.GetSelectStmt().
|
||||
ValuesLists[0].GetList().
|
||||
Items[0].GetAConst().
|
||||
GetVal().
|
||||
GetString_().
|
||||
Str
|
||||
|
||||
return addressString, nil
|
||||
}
|
||||
186
statediff/indexer/database/file/types/schema.go
Normal file
186
statediff/indexer/database/file/types/schema.go
Normal file
@ -0,0 +1,186 @@
|
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package types
|
||||
|
||||
var TableIPLDBlock = Table{
|
||||
`public.blocks`,
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "key", dbType: text},
|
||||
{name: "data", dbType: bytea},
|
||||
},
|
||||
}
|
||||
|
||||
var TableNodeInfo = Table{
|
||||
Name: `public.nodes`,
|
||||
Columns: []column{
|
||||
{name: "genesis_block", dbType: varchar},
|
||||
{name: "network_id", dbType: varchar},
|
||||
{name: "node_id", dbType: varchar},
|
||||
{name: "client_name", dbType: varchar},
|
||||
{name: "chain_id", dbType: integer},
|
||||
},
|
||||
}
|
||||
|
||||
var TableHeader = Table{
|
||||
"eth.header_cids",
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "block_hash", dbType: varchar},
|
||||
{name: "parent_hash", dbType: varchar},
|
||||
{name: "cid", dbType: text},
|
||||
{name: "td", dbType: numeric},
|
||||
{name: "node_id", dbType: varchar},
|
||||
{name: "reward", dbType: numeric},
|
||||
{name: "state_root", dbType: varchar},
|
||||
{name: "tx_root", dbType: varchar},
|
||||
{name: "receipt_root", dbType: varchar},
|
||||
{name: "uncle_root", dbType: varchar},
|
||||
{name: "bloom", dbType: bytea},
|
||||
{name: "timestamp", dbType: numeric},
|
||||
{name: "mh_key", dbType: text},
|
||||
{name: "times_validated", dbType: integer},
|
||||
{name: "coinbase", dbType: varchar},
|
||||
},
|
||||
}
|
||||
|
||||
var TableStateNode = Table{
|
||||
"eth.state_cids",
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "header_id", dbType: varchar},
|
||||
{name: "state_leaf_key", dbType: varchar},
|
||||
{name: "cid", dbType: text},
|
||||
{name: "state_path", dbType: bytea},
|
||||
{name: "node_type", dbType: integer},
|
||||
{name: "diff", dbType: boolean},
|
||||
{name: "mh_key", dbType: text},
|
||||
},
|
||||
}
|
||||
|
||||
var TableStorageNode = Table{
|
||||
"eth.storage_cids",
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "header_id", dbType: varchar},
|
||||
{name: "state_path", dbType: bytea},
|
||||
{name: "storage_leaf_key", dbType: varchar},
|
||||
{name: "cid", dbType: text},
|
||||
{name: "storage_path", dbType: bytea},
|
||||
{name: "node_type", dbType: integer},
|
||||
{name: "diff", dbType: boolean},
|
||||
{name: "mh_key", dbType: text},
|
||||
},
|
||||
}
|
||||
|
||||
var TableUncle = Table{
|
||||
"eth.uncle_cids",
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "block_hash", dbType: varchar},
|
||||
{name: "header_id", dbType: varchar},
|
||||
{name: "parent_hash", dbType: varchar},
|
||||
{name: "cid", dbType: text},
|
||||
{name: "reward", dbType: numeric},
|
||||
{name: "mh_key", dbType: text},
|
||||
},
|
||||
}
|
||||
|
||||
var TableTransaction = Table{
|
||||
"eth.transaction_cids",
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "header_id", dbType: varchar},
|
||||
{name: "tx_hash", dbType: varchar},
|
||||
{name: "cid", dbType: text},
|
||||
{name: "dst", dbType: varchar},
|
||||
{name: "src", dbType: varchar},
|
||||
{name: "index", dbType: integer},
|
||||
{name: "mh_key", dbType: text},
|
||||
{name: "tx_data", dbType: bytea},
|
||||
{name: "tx_type", dbType: integer},
|
||||
{name: "value", dbType: numeric},
|
||||
},
|
||||
}
|
||||
|
||||
var TableAccessListElement = Table{
|
||||
"eth.access_list_elements",
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "tx_id", dbType: varchar},
|
||||
{name: "index", dbType: integer},
|
||||
{name: "address", dbType: varchar},
|
||||
{name: "storage_keys", dbType: varchar, isArray: true},
|
||||
},
|
||||
}
|
||||
|
||||
var TableReceipt = Table{
|
||||
"eth.receipt_cids",
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "header_id", dbType: varchar},
|
||||
{name: "tx_id", dbType: varchar},
|
||||
{name: "leaf_cid", dbType: text},
|
||||
{name: "contract", dbType: varchar},
|
||||
{name: "contract_hash", dbType: varchar},
|
||||
{name: "leaf_mh_key", dbType: text},
|
||||
{name: "post_state", dbType: varchar},
|
||||
{name: "post_status", dbType: integer},
|
||||
{name: "log_root", dbType: varchar},
|
||||
},
|
||||
}
|
||||
|
||||
var TableLog = Table{
|
||||
"eth.log_cids",
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "header_id", dbType: varchar},
|
||||
{name: "leaf_cid", dbType: text},
|
||||
{name: "leaf_mh_key", dbType: text},
|
||||
{name: "rct_id", dbType: varchar},
|
||||
{name: "address", dbType: varchar},
|
||||
{name: "index", dbType: integer},
|
||||
{name: "topic0", dbType: varchar},
|
||||
{name: "topic1", dbType: varchar},
|
||||
{name: "topic2", dbType: varchar},
|
||||
{name: "topic3", dbType: varchar},
|
||||
{name: "log_data", dbType: bytea},
|
||||
},
|
||||
}
|
||||
|
||||
var TableStateAccount = Table{
|
||||
"eth.state_accounts",
|
||||
[]column{
|
||||
{name: "block_number", dbType: bigint},
|
||||
{name: "header_id", dbType: varchar},
|
||||
{name: "state_path", dbType: bytea},
|
||||
{name: "balance", dbType: numeric},
|
||||
{name: "nonce", dbType: bigint},
|
||||
{name: "code_hash", dbType: bytea},
|
||||
{name: "storage_root", dbType: varchar},
|
||||
},
|
||||
}
|
||||
|
||||
var TableWatchedAddresses = Table{
|
||||
"eth_meta.watched_addresses",
|
||||
[]column{
|
||||
{name: "address", dbType: varchar},
|
||||
{name: "created_at", dbType: bigint},
|
||||
{name: "watched_at", dbType: bigint},
|
||||
{name: "last_filled_at", dbType: bigint},
|
||||
},
|
||||
}
|
||||
104
statediff/indexer/database/file/types/table.go
Normal file
104
statediff/indexer/database/file/types/table.go
Normal file
@ -0,0 +1,104 @@
|
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package types
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/thoas/go-funk"
|
||||
)
|
||||
|
||||
type colType int
|
||||
|
||||
const (
|
||||
integer colType = iota
|
||||
boolean
|
||||
bigint
|
||||
numeric
|
||||
bytea
|
||||
varchar
|
||||
text
|
||||
)
|
||||
|
||||
type column struct {
|
||||
name string
|
||||
dbType colType
|
||||
isArray bool
|
||||
}
|
||||
type Table struct {
|
||||
Name string
|
||||
Columns []column
|
||||
}
|
||||
|
||||
func (tbl *Table) ToCsvRow(args ...interface{}) []string {
|
||||
var row []string
|
||||
for i, col := range tbl.Columns {
|
||||
value := col.dbType.formatter()(args[i])
|
||||
|
||||
if col.isArray {
|
||||
valueList := funk.Map(args[i], col.dbType.formatter()).([]string)
|
||||
value = fmt.Sprintf("{%s}", strings.Join(valueList, ","))
|
||||
}
|
||||
|
||||
row = append(row, value)
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func (tbl *Table) VarcharColumns() []string {
|
||||
columns := funk.Filter(tbl.Columns, func(col column) bool {
|
||||
return col.dbType == varchar
|
||||
}).([]column)
|
||||
|
||||
columnNames := funk.Map(columns, func(col column) string {
|
||||
return col.name
|
||||
}).([]string)
|
||||
|
||||
return columnNames
|
||||
}
|
||||
|
||||
type colfmt = func(interface{}) string
|
||||
|
||||
func sprintf(f string) colfmt {
|
||||
return func(x interface{}) string { return fmt.Sprintf(f, x) }
|
||||
}
|
||||
|
||||
func (typ colType) formatter() colfmt {
|
||||
switch typ {
|
||||
case integer:
|
||||
return sprintf("%d")
|
||||
case boolean:
|
||||
return func(x interface{}) string {
|
||||
if x.(bool) {
|
||||
return "t"
|
||||
}
|
||||
return "f"
|
||||
}
|
||||
case bigint:
|
||||
return sprintf("%s")
|
||||
case numeric:
|
||||
return sprintf("%s")
|
||||
case bytea:
|
||||
return sprintf(`\x%x`)
|
||||
case varchar:
|
||||
return sprintf("%s")
|
||||
case text:
|
||||
return sprintf("%s")
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
263
statediff/indexer/database/metrics/metrics.go
Normal file
263
statediff/indexer/database/metrics/metrics.go
Normal file
@ -0,0 +1,263 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
)
|
||||
|
||||
const (
|
||||
namespace = "statediff"
|
||||
)
|
||||
|
||||
var (
|
||||
IndexerMetrics = RegisterIndexerMetrics(metrics.DefaultRegistry)
|
||||
DBMetrics = RegisterDBMetrics(metrics.DefaultRegistry)
|
||||
)
|
||||
|
||||
// Build a fully qualified metric name
|
||||
func metricName(subsystem, name string) string {
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
parts := []string{namespace, name}
|
||||
if subsystem != "" {
|
||||
parts = []string{namespace, subsystem, name}
|
||||
}
|
||||
// Prometheus uses _ but geth metrics uses / and replaces
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
type IndexerMetricsHandles struct {
|
||||
// The total number of processed BlocksCounter
|
||||
BlocksCounter metrics.Counter
|
||||
// The total number of processed transactions
|
||||
TransactionsCounter metrics.Counter
|
||||
// The total number of processed receipts
|
||||
ReceiptsCounter metrics.Counter
|
||||
// The total number of processed logs
|
||||
LogsCounter metrics.Counter
|
||||
// The total number of access list entries processed
|
||||
AccessListEntriesCounter metrics.Counter
|
||||
// Time spent waiting for free postgres tx
|
||||
FreePostgresTimer metrics.Timer
|
||||
// Postgres transaction commit duration
|
||||
PostgresCommitTimer metrics.Timer
|
||||
// Header processing time
|
||||
HeaderProcessingTimer metrics.Timer
|
||||
// Uncle processing time
|
||||
UncleProcessingTimer metrics.Timer
|
||||
// Tx and receipt processing time
|
||||
TxAndRecProcessingTimer metrics.Timer
|
||||
// State, storage, and code combined processing time
|
||||
StateStoreCodeProcessingTimer metrics.Timer
|
||||
|
||||
// Fine-grained code timers
|
||||
BuildStateDiffWithIntermediateStateNodesTimer metrics.Timer
|
||||
BuildStateDiffWithoutIntermediateStateNodesTimer metrics.Timer
|
||||
CreatedAndUpdatedStateWithIntermediateNodesTimer metrics.Timer
|
||||
DeletedOrUpdatedStateTimer metrics.Timer
|
||||
BuildAccountUpdatesTimer metrics.Timer
|
||||
BuildAccountCreationsTimer metrics.Timer
|
||||
ResolveNodeTimer metrics.Timer
|
||||
SortKeysTimer metrics.Timer
|
||||
FindIntersectionTimer metrics.Timer
|
||||
OutputTimer metrics.Timer
|
||||
CodeOutputTimer metrics.Timer
|
||||
DifferenceIteratorNextTimer metrics.Timer
|
||||
DifferenceIteratorCounter metrics.Counter
|
||||
DeletedOrUpdatedStorageTimer metrics.Timer
|
||||
CreatedAndUpdatedStorageTimer metrics.Timer
|
||||
BuildStorageNodesIncrementalTimer metrics.Timer
|
||||
BuildStateTrieObjectTimer metrics.Timer
|
||||
BuildStateTrieTimer metrics.Timer
|
||||
BuildStateDiffObjectTimer metrics.Timer
|
||||
WriteStateDiffObjectTimer metrics.Timer
|
||||
CreatedAndUpdatedStateTimer metrics.Timer
|
||||
BuildStorageNodesEventualTimer metrics.Timer
|
||||
BuildStorageNodesFromTrieTimer metrics.Timer
|
||||
BuildRemovedAccountStorageNodesTimer metrics.Timer
|
||||
BuildRemovedStorageNodesFromTrieTimer metrics.Timer
|
||||
IsWatchedAddressTimer metrics.Timer
|
||||
}
|
||||
|
||||
func RegisterIndexerMetrics(reg metrics.Registry) IndexerMetricsHandles {
|
||||
ctx := IndexerMetricsHandles{
|
||||
BlocksCounter: metrics.NewCounter(),
|
||||
TransactionsCounter: metrics.NewCounter(),
|
||||
ReceiptsCounter: metrics.NewCounter(),
|
||||
LogsCounter: metrics.NewCounter(),
|
||||
AccessListEntriesCounter: metrics.NewCounter(),
|
||||
FreePostgresTimer: metrics.NewTimer(),
|
||||
PostgresCommitTimer: metrics.NewTimer(),
|
||||
HeaderProcessingTimer: metrics.NewTimer(),
|
||||
UncleProcessingTimer: metrics.NewTimer(),
|
||||
TxAndRecProcessingTimer: metrics.NewTimer(),
|
||||
StateStoreCodeProcessingTimer: metrics.NewTimer(),
|
||||
BuildStateDiffWithIntermediateStateNodesTimer: metrics.NewTimer(),
|
||||
BuildStateDiffWithoutIntermediateStateNodesTimer: metrics.NewTimer(),
|
||||
CreatedAndUpdatedStateWithIntermediateNodesTimer: metrics.NewTimer(),
|
||||
DeletedOrUpdatedStateTimer: metrics.NewTimer(),
|
||||
BuildAccountUpdatesTimer: metrics.NewTimer(),
|
||||
BuildAccountCreationsTimer: metrics.NewTimer(),
|
||||
ResolveNodeTimer: metrics.NewTimer(),
|
||||
SortKeysTimer: metrics.NewTimer(),
|
||||
FindIntersectionTimer: metrics.NewTimer(),
|
||||
OutputTimer: metrics.NewTimer(),
|
||||
CodeOutputTimer: metrics.NewTimer(),
|
||||
DifferenceIteratorNextTimer: metrics.NewTimer(),
|
||||
DifferenceIteratorCounter: metrics.NewCounter(),
|
||||
DeletedOrUpdatedStorageTimer: metrics.NewTimer(),
|
||||
CreatedAndUpdatedStorageTimer: metrics.NewTimer(),
|
||||
BuildStorageNodesIncrementalTimer: metrics.NewTimer(),
|
||||
BuildStateTrieObjectTimer: metrics.NewTimer(),
|
||||
BuildStateTrieTimer: metrics.NewTimer(),
|
||||
BuildStateDiffObjectTimer: metrics.NewTimer(),
|
||||
WriteStateDiffObjectTimer: metrics.NewTimer(),
|
||||
CreatedAndUpdatedStateTimer: metrics.NewTimer(),
|
||||
BuildStorageNodesEventualTimer: metrics.NewTimer(),
|
||||
BuildStorageNodesFromTrieTimer: metrics.NewTimer(),
|
||||
BuildRemovedAccountStorageNodesTimer: metrics.NewTimer(),
|
||||
BuildRemovedStorageNodesFromTrieTimer: metrics.NewTimer(),
|
||||
IsWatchedAddressTimer: metrics.NewTimer(),
|
||||
}
|
||||
subsys := "indexer"
|
||||
reg.Register(metricName(subsys, "blocks"), ctx.BlocksCounter)
|
||||
reg.Register(metricName(subsys, "transactions"), ctx.TransactionsCounter)
|
||||
reg.Register(metricName(subsys, "receipts"), ctx.ReceiptsCounter)
|
||||
reg.Register(metricName(subsys, "logs"), ctx.LogsCounter)
|
||||
reg.Register(metricName(subsys, "access_list_entries"), ctx.AccessListEntriesCounter)
|
||||
reg.Register(metricName(subsys, "t_free_postgres"), ctx.FreePostgresTimer)
|
||||
reg.Register(metricName(subsys, "t_postgres_commit"), ctx.PostgresCommitTimer)
|
||||
reg.Register(metricName(subsys, "t_header_processing"), ctx.HeaderProcessingTimer)
|
||||
reg.Register(metricName(subsys, "t_uncle_processing"), ctx.UncleProcessingTimer)
|
||||
reg.Register(metricName(subsys, "t_tx_receipt_processing"), ctx.TxAndRecProcessingTimer)
|
||||
reg.Register(metricName(subsys, "t_state_store_code_processing"), ctx.StateStoreCodeProcessingTimer)
|
||||
reg.Register(metricName(subsys, "t_build_statediff_with_intermediate_state_nodes"), ctx.BuildStateDiffWithIntermediateStateNodesTimer)
|
||||
reg.Register(metricName(subsys, "t_build_statediff_without_intermediate_state_nodes"), ctx.BuildStateDiffWithoutIntermediateStateNodesTimer)
|
||||
reg.Register(metricName(subsys, "t_created_and_update_state_with_intermediate_nodes"), ctx.CreatedAndUpdatedStateWithIntermediateNodesTimer)
|
||||
reg.Register(metricName(subsys, "t_deleted_or_updated_state"), ctx.DeletedOrUpdatedStateTimer)
|
||||
reg.Register(metricName(subsys, "t_build_account_updates"), ctx.BuildAccountUpdatesTimer)
|
||||
reg.Register(metricName(subsys, "t_build_account_creations"), ctx.BuildAccountCreationsTimer)
|
||||
reg.Register(metricName(subsys, "t_resolve_node"), ctx.ResolveNodeTimer)
|
||||
reg.Register(metricName(subsys, "t_sort_keys"), ctx.SortKeysTimer)
|
||||
reg.Register(metricName(subsys, "t_find_intersection"), ctx.FindIntersectionTimer)
|
||||
reg.Register(metricName(subsys, "t_output_fn"), ctx.OutputTimer)
|
||||
reg.Register(metricName(subsys, "t_code_output_fn"), ctx.CodeOutputTimer)
|
||||
reg.Register(metricName(subsys, "t_difference_iterator_next"), ctx.DifferenceIteratorNextTimer)
|
||||
reg.Register(metricName(subsys, "difference_iterator_counter"), ctx.DifferenceIteratorCounter)
|
||||
reg.Register(metricName(subsys, "t_created_and_updated_storage"), ctx.CreatedAndUpdatedStorageTimer)
|
||||
reg.Register(metricName(subsys, "t_deleted_or_updated_storage"), ctx.DeletedOrUpdatedStorageTimer)
|
||||
reg.Register(metricName(subsys, "t_build_storage_nodes_incremental"), ctx.BuildStorageNodesIncrementalTimer)
|
||||
reg.Register(metricName(subsys, "t_build_state_trie_object"), ctx.BuildStateTrieObjectTimer)
|
||||
reg.Register(metricName(subsys, "t_build_state_trie"), ctx.BuildStateTrieTimer)
|
||||
reg.Register(metricName(subsys, "t_build_statediff_object"), ctx.BuildStateDiffObjectTimer)
|
||||
reg.Register(metricName(subsys, "t_write_statediff_object"), ctx.WriteStateDiffObjectTimer)
|
||||
reg.Register(metricName(subsys, "t_created_and_updated_state"), ctx.CreatedAndUpdatedStateTimer)
|
||||
reg.Register(metricName(subsys, "t_build_storage_nodes_eventual"), ctx.BuildStorageNodesEventualTimer)
|
||||
reg.Register(metricName(subsys, "t_build_storage_nodes_from_trie"), ctx.BuildStorageNodesFromTrieTimer)
|
||||
reg.Register(metricName(subsys, "t_build_removed_accounts_storage_nodes"), ctx.BuildRemovedAccountStorageNodesTimer)
|
||||
reg.Register(metricName(subsys, "t_build_removed_storage_nodes_from_trie"), ctx.BuildRemovedStorageNodesFromTrieTimer)
|
||||
reg.Register(metricName(subsys, "t_is_watched_address"), ctx.IsWatchedAddressTimer)
|
||||
|
||||
log.Debug("Registering statediff indexer metrics.")
|
||||
return ctx
|
||||
}
|
||||
|
||||
type dbMetricsHandles struct {
|
||||
// Maximum number of open connections to the sql
|
||||
maxOpen metrics.Gauge
|
||||
// The number of established connections both in use and idle
|
||||
open metrics.Gauge
|
||||
// The number of connections currently in use
|
||||
inUse metrics.Gauge
|
||||
// The number of idle connections
|
||||
idle metrics.Gauge
|
||||
// The total number of connections waited for
|
||||
waitedFor metrics.Counter
|
||||
// The total time blocked waiting for a new connection
|
||||
blockedMilliseconds metrics.Counter
|
||||
// The total number of connections closed due to SetMaxIdleConns
|
||||
closedMaxIdle metrics.Counter
|
||||
// The total number of connections closed due to SetConnMaxLifetime
|
||||
closedMaxLifetime metrics.Counter
|
||||
}
|
||||
|
||||
func RegisterDBMetrics(reg metrics.Registry) dbMetricsHandles {
|
||||
ctx := dbMetricsHandles{
|
||||
maxOpen: metrics.NewGauge(),
|
||||
open: metrics.NewGauge(),
|
||||
inUse: metrics.NewGauge(),
|
||||
idle: metrics.NewGauge(),
|
||||
waitedFor: metrics.NewCounter(),
|
||||
blockedMilliseconds: metrics.NewCounter(),
|
||||
closedMaxIdle: metrics.NewCounter(),
|
||||
closedMaxLifetime: metrics.NewCounter(),
|
||||
}
|
||||
subsys := "connections"
|
||||
reg.Register(metricName(subsys, "max_open"), ctx.maxOpen)
|
||||
reg.Register(metricName(subsys, "open"), ctx.open)
|
||||
reg.Register(metricName(subsys, "in_use"), ctx.inUse)
|
||||
reg.Register(metricName(subsys, "idle"), ctx.idle)
|
||||
reg.Register(metricName(subsys, "waited_for"), ctx.waitedFor)
|
||||
reg.Register(metricName(subsys, "blocked_milliseconds"), ctx.blockedMilliseconds)
|
||||
reg.Register(metricName(subsys, "closed_max_idle"), ctx.closedMaxIdle)
|
||||
reg.Register(metricName(subsys, "closed_max_lifetime"), ctx.closedMaxLifetime)
|
||||
|
||||
log.Debug("Registering statediff DB metrics.")
|
||||
return ctx
|
||||
}
|
||||
|
||||
// DbStats interface to accommodate different concrete sql stats types
|
||||
type DbStats interface {
|
||||
MaxOpen() int64
|
||||
Open() int64
|
||||
InUse() int64
|
||||
Idle() int64
|
||||
WaitCount() int64
|
||||
WaitDuration() time.Duration
|
||||
MaxIdleClosed() int64
|
||||
MaxLifetimeClosed() int64
|
||||
}
|
||||
|
||||
func (met *dbMetricsHandles) Update(stats DbStats) {
|
||||
met.maxOpen.Update(stats.MaxOpen())
|
||||
met.open.Update(stats.Open())
|
||||
met.inUse.Update(stats.InUse())
|
||||
met.idle.Update(stats.Idle())
|
||||
met.waitedFor.Inc(stats.WaitCount())
|
||||
met.blockedMilliseconds.Inc(stats.WaitDuration().Milliseconds())
|
||||
met.closedMaxIdle.Inc(stats.MaxIdleClosed())
|
||||
met.closedMaxLifetime.Inc(stats.MaxLifetimeClosed())
|
||||
}
|
||||
|
||||
func ReportAndUpdateDuration(msg string, start time.Time, logger log.Logger, timer metrics.Timer) {
|
||||
since := UpdateDuration(start, timer)
|
||||
logger.Debug(fmt.Sprintf("%s duration=%dms", msg, since.Milliseconds()))
|
||||
}
|
||||
|
||||
func UpdateDuration(start time.Time, timer metrics.Timer) time.Duration {
|
||||
since := time.Since(start)
|
||||
timer.Update(since)
|
||||
return since
|
||||
}
|
||||
144
statediff/indexer/database/sql/batch_tx.go
Normal file
144
statediff/indexer/database/sql/batch_tx.go
Normal file
@ -0,0 +1,144 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
dshelp "github.com/ipfs/go-ipfs-ds-help"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
"github.com/lib/pq"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/models"
|
||||
)
|
||||
|
||||
const startingCacheCapacity = 1024 * 24
|
||||
|
||||
// BatchTx wraps a sql tx with the state necessary for building the tx concurrently during trie difference iteration
|
||||
type BatchTx struct {
|
||||
BlockNumber string
|
||||
ctx context.Context
|
||||
dbtx Tx
|
||||
stm string
|
||||
quit chan struct{}
|
||||
iplds chan models.IPLDModel
|
||||
ipldCache models.IPLDBatch
|
||||
removedCacheFlag *uint32
|
||||
// Tracks expected cache size and ensures cache is caught up before flush
|
||||
cacheWg sync.WaitGroup
|
||||
|
||||
submit func(blockTx *BatchTx, err error) error
|
||||
}
|
||||
|
||||
// Submit satisfies indexer.AtomicTx
|
||||
func (tx *BatchTx) Submit(err error) error {
|
||||
return tx.submit(tx, err)
|
||||
}
|
||||
|
||||
func (tx *BatchTx) flush() error {
|
||||
tx.cacheWg.Wait()
|
||||
_, err := tx.dbtx.Exec(tx.ctx, tx.stm, pq.Array(tx.ipldCache.BlockNumbers), pq.Array(tx.ipldCache.Keys),
|
||||
pq.Array(tx.ipldCache.Values))
|
||||
if err != nil {
|
||||
log.Debug(insertError{"public.blocks", err, tx.stm,
|
||||
struct {
|
||||
blockNumbers []string
|
||||
keys []string
|
||||
values [][]byte
|
||||
}{
|
||||
tx.ipldCache.BlockNumbers,
|
||||
tx.ipldCache.Keys,
|
||||
tx.ipldCache.Values,
|
||||
}}.Error())
|
||||
return insertError{"public.blocks", err, tx.stm, "too many arguments; use debug mode for full list"}
|
||||
}
|
||||
tx.ipldCache = models.IPLDBatch{}
|
||||
return nil
|
||||
}
|
||||
|
||||
// run in background goroutine to synchronize concurrent appends to the ipldCache
|
||||
func (tx *BatchTx) cache() {
|
||||
for {
|
||||
select {
|
||||
case i := <-tx.iplds:
|
||||
tx.ipldCache.BlockNumbers = append(tx.ipldCache.BlockNumbers, i.BlockNumber)
|
||||
tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
|
||||
tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
|
||||
tx.cacheWg.Done()
|
||||
case <-tx.quit:
|
||||
tx.ipldCache = models.IPLDBatch{}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (tx *BatchTx) cacheDirect(key string, value []byte) {
|
||||
tx.cacheWg.Add(1)
|
||||
tx.iplds <- models.IPLDModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
Key: key,
|
||||
Data: value,
|
||||
}
|
||||
}
|
||||
|
||||
func (tx *BatchTx) cacheIPLD(i node.Node) {
|
||||
tx.cacheWg.Add(1)
|
||||
tx.iplds <- models.IPLDModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(),
|
||||
Data: i.RawData(),
|
||||
}
|
||||
}
|
||||
|
||||
func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error) {
|
||||
c, err := ipld.RawdataToCid(codec, raw, mh)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
|
||||
tx.cacheWg.Add(1)
|
||||
tx.iplds <- models.IPLDModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
Key: prefixedKey,
|
||||
Data: raw,
|
||||
}
|
||||
return c.String(), prefixedKey, err
|
||||
}
|
||||
|
||||
func (tx *BatchTx) cacheRemoved(key string, value []byte) {
|
||||
if atomic.LoadUint32(tx.removedCacheFlag) == 0 {
|
||||
atomic.StoreUint32(tx.removedCacheFlag, 1)
|
||||
tx.cacheWg.Add(1)
|
||||
tx.iplds <- models.IPLDModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
Key: key,
|
||||
Data: value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rollback sql transaction and log any error
|
||||
func rollback(ctx context.Context, tx Tx) {
|
||||
if err := tx.Rollback(ctx); err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
}
|
||||
671
statediff/indexer/database/sql/indexer.go
Normal file
671
statediff/indexer/database/sql/indexer.go
Normal file
@ -0,0 +1,671 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Package sql provides an interface for pushing and indexing IPLD objects into a sql database
|
||||
// Metrics for reporting processing and connection stats are defined in ./metrics.go
|
||||
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
"github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
metrics2 "github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/models"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
|
||||
)
|
||||
|
||||
var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
|
||||
|
||||
// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of an SQL sql
|
||||
type StateDiffIndexer struct {
|
||||
ctx context.Context
|
||||
chainConfig *params.ChainConfig
|
||||
dbWriter *Writer
|
||||
}
|
||||
|
||||
// NewStateDiffIndexer creates a sql implementation of interfaces.StateDiffIndexer
|
||||
func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, db Database) (*StateDiffIndexer, error) {
|
||||
return &StateDiffIndexer{
|
||||
ctx: ctx,
|
||||
chainConfig: chainConfig,
|
||||
dbWriter: NewWriter(db),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ReportDBMetrics is a reporting function to run as goroutine
|
||||
func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bool) {
|
||||
if !metrics.Enabled {
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(delay)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
metrics2.DBMetrics.Update(sdi.dbWriter.db.Stats())
|
||||
case <-quit:
|
||||
ticker.Stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
|
||||
// Returns an initiated DB transaction which must be Closed via defer to commit or rollback
|
||||
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
|
||||
start, t := time.Now(), time.Now()
|
||||
blockHash := block.Hash()
|
||||
blockHashStr := blockHash.String()
|
||||
height := block.NumberU64()
|
||||
traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
|
||||
transactions := block.Transactions()
|
||||
// Derive any missing fields
|
||||
if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, transactions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Generate the block iplds
|
||||
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
|
||||
}
|
||||
|
||||
if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) {
|
||||
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
|
||||
}
|
||||
if len(txTrieNodes) != len(rctTrieNodes) {
|
||||
return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
|
||||
}
|
||||
|
||||
// Calculate reward
|
||||
var reward *big.Int
|
||||
// in PoA networks block reward is 0
|
||||
if sdi.chainConfig.Clique != nil {
|
||||
reward = big.NewInt(0)
|
||||
} else {
|
||||
reward = shared.CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts)
|
||||
}
|
||||
t = time.Now()
|
||||
|
||||
// Begin new DB tx for everything
|
||||
tx := NewDelayedTx(sdi.dbWriter.db)
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
rollback(sdi.ctx, tx)
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
rollback(sdi.ctx, tx)
|
||||
}
|
||||
}()
|
||||
blockTx := &BatchTx{
|
||||
removedCacheFlag: new(uint32),
|
||||
ctx: sdi.ctx,
|
||||
BlockNumber: block.Number().String(),
|
||||
stm: sdi.dbWriter.db.InsertIPLDsStm(),
|
||||
iplds: make(chan models.IPLDModel),
|
||||
quit: make(chan struct{}),
|
||||
ipldCache: models.IPLDBatch{
|
||||
BlockNumbers: make([]string, 0, startingCacheCapacity),
|
||||
Keys: make([]string, 0, startingCacheCapacity),
|
||||
Values: make([][]byte, 0, startingCacheCapacity),
|
||||
},
|
||||
dbtx: tx,
|
||||
// handle transaction commit or rollback for any return case
|
||||
submit: func(self *BatchTx, err error) error {
|
||||
defer func() {
|
||||
close(self.quit)
|
||||
close(self.iplds)
|
||||
}()
|
||||
if p := recover(); p != nil {
|
||||
log.Info("panic detected before tx submission, rolling back the tx", "panic", p)
|
||||
rollback(sdi.ctx, tx)
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
log.Info("error detected before tx submission, rolling back the tx", "error", err)
|
||||
rollback(sdi.ctx, tx)
|
||||
} else {
|
||||
tDiff := time.Since(t)
|
||||
metrics2.IndexerMetrics.StateStoreCodeProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
if err := self.flush(); err != nil {
|
||||
rollback(sdi.ctx, tx)
|
||||
traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
|
||||
log.Debug(traceMsg)
|
||||
return err
|
||||
}
|
||||
err = tx.Commit(sdi.ctx)
|
||||
tDiff = time.Since(t)
|
||||
metrics2.IndexerMetrics.PostgresCommitTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String())
|
||||
}
|
||||
traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String())
|
||||
log.Debug(traceMsg)
|
||||
return err
|
||||
},
|
||||
}
|
||||
go blockTx.cache()
|
||||
|
||||
tDiff := time.Since(t)
|
||||
metrics2.IndexerMetrics.FreePostgresTimer.Update(tDiff)
|
||||
|
||||
traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
|
||||
// Publish and index header, collect headerID
|
||||
var headerID string
|
||||
headerID, err = sdi.processHeader(blockTx, block.Header(), headerNode, reward, totalDifficulty)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tDiff = time.Since(t)
|
||||
metrics2.IndexerMetrics.HeaderProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
// Publish and index uncles
|
||||
err = sdi.processUncles(blockTx, headerID, block.Number(), uncleNodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tDiff = time.Since(t)
|
||||
metrics2.IndexerMetrics.UncleProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
// Publish and index receipts and txs
|
||||
err = sdi.processReceiptsAndTxs(blockTx, processArgs{
|
||||
headerID: headerID,
|
||||
blockNumber: block.Number(),
|
||||
receipts: receipts,
|
||||
txs: transactions,
|
||||
rctNodes: rctNodes,
|
||||
rctTrieNodes: rctTrieNodes,
|
||||
txNodes: txNodes,
|
||||
txTrieNodes: txTrieNodes,
|
||||
logTrieNodes: logTrieNodes,
|
||||
logLeafNodeCIDs: logLeafNodeCIDs,
|
||||
rctLeafNodeCIDs: rctLeafNodeCIDs,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tDiff = time.Since(t)
|
||||
metrics2.IndexerMetrics.TxAndRecProcessingTimer.Update(tDiff)
|
||||
traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String())
|
||||
t = time.Now()
|
||||
|
||||
return blockTx, err
|
||||
}
|
||||
|
||||
// processHeader publishes and indexes a header IPLD in Postgres
|
||||
// it returns the headerID
|
||||
func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode node.Node, reward, td *big.Int) (string, error) {
|
||||
tx.cacheIPLD(headerNode)
|
||||
|
||||
var baseFee *string
|
||||
if header.BaseFee != nil {
|
||||
baseFee = new(string)
|
||||
*baseFee = header.BaseFee.String()
|
||||
}
|
||||
headerID := header.Hash().String()
|
||||
// index header
|
||||
return headerID, sdi.dbWriter.upsertHeaderCID(tx.dbtx, models.HeaderModel{
|
||||
CID: headerNode.Cid().String(),
|
||||
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
|
||||
ParentHash: header.ParentHash.String(),
|
||||
BlockNumber: header.Number.String(),
|
||||
BlockHash: headerID,
|
||||
TotalDifficulty: td.String(),
|
||||
Reward: reward.String(),
|
||||
Bloom: header.Bloom.Bytes(),
|
||||
StateRoot: header.Root.String(),
|
||||
RctRoot: header.ReceiptHash.String(),
|
||||
TxRoot: header.TxHash.String(),
|
||||
UncleRoot: header.UncleHash.String(),
|
||||
Timestamp: header.Time,
|
||||
Coinbase: header.Coinbase.String(),
|
||||
})
|
||||
}
|
||||
|
||||
// processUncles publishes and indexes uncle IPLDs in Postgres
|
||||
func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, uncleNodes []*ipld2.EthHeader) error {
|
||||
// publish and index uncles
|
||||
for _, uncleNode := range uncleNodes {
|
||||
tx.cacheIPLD(uncleNode)
|
||||
var uncleReward *big.Int
|
||||
// in PoA networks uncle reward is 0
|
||||
if sdi.chainConfig.Clique != nil {
|
||||
uncleReward = big.NewInt(0)
|
||||
} else {
|
||||
uncleReward = shared.CalcUncleMinerReward(blockNumber.Uint64(), uncleNode.Number.Uint64())
|
||||
}
|
||||
uncle := models.UncleModel{
|
||||
BlockNumber: blockNumber.String(),
|
||||
HeaderID: headerID,
|
||||
CID: uncleNode.Cid().String(),
|
||||
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
|
||||
ParentHash: uncleNode.ParentHash.String(),
|
||||
BlockHash: uncleNode.Hash().String(),
|
||||
Reward: uncleReward.String(),
|
||||
}
|
||||
if err := sdi.dbWriter.upsertUncleCID(tx.dbtx, uncle); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// processArgs bundles arguments to processReceiptsAndTxs
|
||||
type processArgs struct {
|
||||
headerID string
|
||||
blockNumber *big.Int
|
||||
receipts types.Receipts
|
||||
txs types.Transactions
|
||||
rctNodes []*ipld2.EthReceipt
|
||||
rctTrieNodes []*ipld2.EthRctTrie
|
||||
txNodes []*ipld2.EthTx
|
||||
txTrieNodes []*ipld2.EthTxTrie
|
||||
logTrieNodes [][]node.Node
|
||||
logLeafNodeCIDs [][]cid.Cid
|
||||
rctLeafNodeCIDs []cid.Cid
|
||||
}
|
||||
|
||||
// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
|
||||
func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error {
|
||||
// Process receipts and txs
|
||||
signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
|
||||
for i, receipt := range args.receipts {
|
||||
for _, logTrieNode := range args.logTrieNodes[i] {
|
||||
tx.cacheIPLD(logTrieNode)
|
||||
}
|
||||
txNode := args.txNodes[i]
|
||||
tx.cacheIPLD(txNode)
|
||||
|
||||
// index tx
|
||||
trx := args.txs[i]
|
||||
txID := trx.Hash().String()
|
||||
|
||||
var val string
|
||||
if trx.Value() != nil {
|
||||
val = trx.Value().String()
|
||||
}
|
||||
|
||||
// derive sender for the tx that corresponds with this receipt
|
||||
from, err := types.Sender(signer, trx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deriving tx sender: %v", err)
|
||||
}
|
||||
txModel := models.TxModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
HeaderID: args.headerID,
|
||||
Dst: shared.HandleZeroAddrPointer(trx.To()),
|
||||
Src: shared.HandleZeroAddr(from),
|
||||
TxHash: txID,
|
||||
Index: int64(i),
|
||||
Data: trx.Data(),
|
||||
CID: txNode.Cid().String(),
|
||||
MhKey: shared.MultihashKeyFromCID(txNode.Cid()),
|
||||
Type: trx.Type(),
|
||||
Value: val,
|
||||
}
|
||||
if err := sdi.dbWriter.upsertTransactionCID(tx.dbtx, txModel); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// index access list if this is one
|
||||
for j, accessListElement := range trx.AccessList() {
|
||||
storageKeys := make([]string, len(accessListElement.StorageKeys))
|
||||
for k, storageKey := range accessListElement.StorageKeys {
|
||||
storageKeys[k] = storageKey.Hex()
|
||||
}
|
||||
accessListElementModel := models.AccessListElementModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
TxID: txID,
|
||||
Index: int64(j),
|
||||
Address: accessListElement.Address.Hex(),
|
||||
StorageKeys: storageKeys,
|
||||
}
|
||||
if err := sdi.dbWriter.upsertAccessListElement(tx.dbtx, accessListElementModel); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// this is the contract address if this receipt is for a contract creation tx
|
||||
contract := shared.HandleZeroAddr(receipt.ContractAddress)
|
||||
var contractHash string
|
||||
if contract != "" {
|
||||
contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
|
||||
}
|
||||
|
||||
// index receipt
|
||||
if !args.rctLeafNodeCIDs[i].Defined() {
|
||||
return fmt.Errorf("invalid receipt leaf node cid")
|
||||
}
|
||||
|
||||
rctModel := &models.ReceiptModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
HeaderID: args.headerID,
|
||||
TxID: txID,
|
||||
Contract: contract,
|
||||
ContractHash: contractHash,
|
||||
LeafCID: args.rctLeafNodeCIDs[i].String(),
|
||||
LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]),
|
||||
LogRoot: args.rctNodes[i].LogRoot.String(),
|
||||
}
|
||||
if len(receipt.PostState) == 0 {
|
||||
rctModel.PostStatus = receipt.Status
|
||||
} else {
|
||||
rctModel.PostState = common.Bytes2Hex(receipt.PostState)
|
||||
}
|
||||
|
||||
if err := sdi.dbWriter.upsertReceiptCID(tx.dbtx, rctModel); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// index logs
|
||||
logDataSet := make([]*models.LogsModel, len(receipt.Logs))
|
||||
for idx, l := range receipt.Logs {
|
||||
topicSet := make([]string, 4)
|
||||
for ti, topic := range l.Topics {
|
||||
topicSet[ti] = topic.Hex()
|
||||
}
|
||||
|
||||
if !args.logLeafNodeCIDs[i][idx].Defined() {
|
||||
return fmt.Errorf("invalid log cid")
|
||||
}
|
||||
|
||||
logDataSet[idx] = &models.LogsModel{
|
||||
BlockNumber: args.blockNumber.String(),
|
||||
HeaderID: args.headerID,
|
||||
ReceiptID: txID,
|
||||
Address: l.Address.String(),
|
||||
Index: int64(l.Index),
|
||||
Data: l.Data,
|
||||
LeafCID: args.logLeafNodeCIDs[i][idx].String(),
|
||||
LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]),
|
||||
Topic0: topicSet[0],
|
||||
Topic1: topicSet[1],
|
||||
Topic2: topicSet[2],
|
||||
Topic3: topicSet[3],
|
||||
}
|
||||
}
|
||||
|
||||
if err := sdi.dbWriter.upsertLogCID(tx.dbtx, logDataSet); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// publish trie nodes, these aren't indexed directly
|
||||
for i, n := range args.txTrieNodes {
|
||||
tx.cacheIPLD(n)
|
||||
tx.cacheIPLD(args.rctTrieNodes[i])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PushStateNode publishes and indexes a state diff node object (including any child storage nodes) in the IPLD sql
|
||||
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error {
|
||||
tx, ok := batch.(*BatchTx)
|
||||
if !ok {
|
||||
return fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
|
||||
}
|
||||
// publish the state node
|
||||
var stateModel models.StateNodeModel
|
||||
if stateNode.NodeType == sdtypes.Removed {
|
||||
tx.cacheRemoved(shared.RemovedNodeMhKey, []byte{})
|
||||
stateModel = models.StateNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
Path: stateNode.Path,
|
||||
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
|
||||
CID: shared.RemovedNodeStateCID,
|
||||
MhKey: shared.RemovedNodeMhKey,
|
||||
NodeType: stateNode.NodeType.Int(),
|
||||
}
|
||||
} else {
|
||||
stateCIDStr, stateMhKey, err := tx.cacheRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error generating and cacheing state node IPLD: %v", err)
|
||||
}
|
||||
stateModel = models.StateNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
Path: stateNode.Path,
|
||||
StateKey: common.BytesToHash(stateNode.LeafKey).String(),
|
||||
CID: stateCIDStr,
|
||||
MhKey: stateMhKey,
|
||||
NodeType: stateNode.NodeType.Int(),
|
||||
}
|
||||
}
|
||||
|
||||
// index the state node
|
||||
if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// if we have a leaf, decode and index the account data
|
||||
if stateNode.NodeType == sdtypes.Leaf {
|
||||
var i []interface{}
|
||||
if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil {
|
||||
return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
|
||||
}
|
||||
if len(i) != 2 {
|
||||
return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements")
|
||||
}
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
|
||||
return fmt.Errorf("error decoding state account rlp: %s", err.Error())
|
||||
}
|
||||
accountModel := models.StateAccountModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
StatePath: stateNode.Path,
|
||||
Balance: account.Balance.String(),
|
||||
Nonce: account.Nonce,
|
||||
CodeHash: account.CodeHash,
|
||||
StorageRoot: account.Root.String(),
|
||||
}
|
||||
if err := sdi.dbWriter.upsertStateAccount(tx.dbtx, accountModel); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// if there are any storage nodes associated with this node, publish and index them
|
||||
for _, storageNode := range stateNode.StorageNodes {
|
||||
if storageNode.NodeType == sdtypes.Removed {
|
||||
tx.cacheRemoved(shared.RemovedNodeMhKey, []byte{})
|
||||
storageModel := models.StorageNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
StatePath: stateNode.Path,
|
||||
Path: storageNode.Path,
|
||||
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
|
||||
CID: shared.RemovedNodeStorageCID,
|
||||
MhKey: shared.RemovedNodeMhKey,
|
||||
NodeType: storageNode.NodeType.Int(),
|
||||
}
|
||||
if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
storageCIDStr, storageMhKey, err := tx.cacheRaw(ipld2.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err)
|
||||
}
|
||||
storageModel := models.StorageNodeModel{
|
||||
BlockNumber: tx.BlockNumber,
|
||||
HeaderID: headerID,
|
||||
StatePath: stateNode.Path,
|
||||
Path: storageNode.Path,
|
||||
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
|
||||
CID: storageCIDStr,
|
||||
MhKey: storageMhKey,
|
||||
NodeType: storageNode.NodeType.Int(),
|
||||
}
|
||||
if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PushCodeAndCodeHash publishes code and codehash pairs to the ipld sql
|
||||
func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error {
|
||||
tx, ok := batch.(*BatchTx)
|
||||
if !ok {
|
||||
return fmt.Errorf("sql: batch is expected to be of type %T, got %T", &BatchTx{}, batch)
|
||||
}
|
||||
// codec doesn't matter since db key is multihash-based
|
||||
mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error deriving multihash key from codehash: %v", err)
|
||||
}
|
||||
tx.cacheDirect(mhKey, codeAndCodeHash.Code)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close satisfies io.Closer
|
||||
func (sdi *StateDiffIndexer) Close() error {
|
||||
return sdi.dbWriter.Close()
|
||||
}
|
||||
|
||||
// Update the known gaps table with the gap information.
|
||||
|
||||
// LoadWatchedAddresses reads watched addresses from the database
|
||||
func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
|
||||
addressStrings := make([]string, 0)
|
||||
pgStr := "SELECT address FROM eth_meta.watched_addresses"
|
||||
err := sdi.dbWriter.db.Select(sdi.ctx, &addressStrings, pgStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error loading watched addresses: %v", err)
|
||||
}
|
||||
|
||||
watchedAddresses := []common.Address{}
|
||||
for _, addressString := range addressStrings {
|
||||
watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString))
|
||||
}
|
||||
|
||||
return watchedAddresses, nil
|
||||
}
|
||||
|
||||
// InsertWatchedAddresses inserts the given addresses in the database
|
||||
func (sdi *StateDiffIndexer) InsertWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) (err error) {
|
||||
tx := NewDelayedTx(sdi.dbWriter.db)
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
rollback(sdi.ctx, tx)
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
rollback(sdi.ctx, tx)
|
||||
} else {
|
||||
err = tx.Commit(sdi.ctx)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, arg := range args {
|
||||
_, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`,
|
||||
arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error inserting watched_addresses entry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// RemoveWatchedAddresses removes the given watched addresses from the database
|
||||
func (sdi *StateDiffIndexer) RemoveWatchedAddresses(args []sdtypes.WatchAddressArg) (err error) {
|
||||
tx := NewDelayedTx(sdi.dbWriter.db)
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
rollback(sdi.ctx, tx)
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
rollback(sdi.ctx, tx)
|
||||
} else {
|
||||
err = tx.Commit(sdi.ctx)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, arg := range args {
|
||||
_, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses WHERE address = $1`, arg.Address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error removing watched_addresses entry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// SetWatchedAddresses clears and inserts the given addresses in the database
|
||||
func (sdi *StateDiffIndexer) SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) (err error) {
|
||||
tx := NewDelayedTx(sdi.dbWriter.db)
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
rollback(sdi.ctx, tx)
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
rollback(sdi.ctx, tx)
|
||||
} else {
|
||||
err = tx.Commit(sdi.ctx)
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = tx.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error setting watched_addresses table: %v", err)
|
||||
}
|
||||
|
||||
for _, arg := range args {
|
||||
_, err = tx.Exec(sdi.ctx, `INSERT INTO eth_meta.watched_addresses (address, created_at, watched_at) VALUES ($1, $2, $3) ON CONFLICT (address) DO NOTHING`,
|
||||
arg.Address, arg.CreatedAt, currentBlockNumber.Uint64())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error setting watched_addresses table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// ClearWatchedAddresses clears all the watched addresses from the database
|
||||
func (sdi *StateDiffIndexer) ClearWatchedAddresses() error {
|
||||
_, err := sdi.dbWriter.db.Exec(sdi.ctx, `DELETE FROM eth_meta.watched_addresses`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error clearing watched_addresses table: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
28
statediff/indexer/database/sql/indexer_shared_test.go
Normal file
28
statediff/indexer/database/sql/indexer_shared_test.go
Normal file
@ -0,0 +1,28 @@
|
||||
package sql_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
|
||||
)
|
||||
|
||||
var (
|
||||
db sql.Database
|
||||
err error
|
||||
ind interfaces.StateDiffIndexer
|
||||
)
|
||||
|
||||
func checkTxClosure(t *testing.T, idle, inUse, open int64) {
|
||||
require.Equal(t, idle, db.Stats().Idle())
|
||||
require.Equal(t, inUse, db.Stats().InUse())
|
||||
require.Equal(t, open, db.Stats().Open())
|
||||
}
|
||||
|
||||
func tearDown(t *testing.T) {
|
||||
test_helpers.TearDownDB(t, db)
|
||||
require.NoError(t, ind.Close())
|
||||
}
|
||||
77
statediff/indexer/database/sql/interfaces.go
Normal file
77
statediff/indexer/database/sql/interfaces.go
Normal file
@ -0,0 +1,77 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
)
|
||||
|
||||
// Database interfaces required by the sql indexer
|
||||
type Database interface {
|
||||
Driver
|
||||
Statements
|
||||
}
|
||||
|
||||
// Driver interface has all the methods required by a driver implementation to support the sql indexer
|
||||
type Driver interface {
|
||||
QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
|
||||
Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
|
||||
Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error
|
||||
Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error
|
||||
Begin(ctx context.Context) (Tx, error)
|
||||
Stats() metrics.DbStats
|
||||
NodeID() string
|
||||
Context() context.Context
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Statements interface to accommodate different SQL query syntax
|
||||
type Statements interface {
|
||||
InsertHeaderStm() string
|
||||
InsertUncleStm() string
|
||||
InsertTxStm() string
|
||||
InsertAccessListElementStm() string
|
||||
InsertRctStm() string
|
||||
InsertLogStm() string
|
||||
InsertStateStm() string
|
||||
InsertAccountStm() string
|
||||
InsertStorageStm() string
|
||||
InsertIPLDStm() string
|
||||
InsertIPLDsStm() string
|
||||
InsertKnownGapsStm() string
|
||||
}
|
||||
|
||||
// Tx interface to accommodate different concrete SQL transaction types
|
||||
type Tx interface {
|
||||
QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow
|
||||
Exec(ctx context.Context, sql string, args ...interface{}) (Result, error)
|
||||
Commit(ctx context.Context) error
|
||||
Rollback(ctx context.Context) error
|
||||
}
|
||||
|
||||
// ScannableRow interface to accommodate different concrete row types
|
||||
type ScannableRow interface {
|
||||
Scan(dest ...interface{}) error
|
||||
}
|
||||
|
||||
// Result interface to accommodate different concrete result types
|
||||
type Result interface {
|
||||
RowsAffected() (int64, error)
|
||||
}
|
||||
55
statediff/indexer/database/sql/lazy_tx.go
Normal file
55
statediff/indexer/database/sql/lazy_tx.go
Normal file
@ -0,0 +1,55 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
type DelayedTx struct {
|
||||
cache []cachedStmt
|
||||
db Database
|
||||
}
|
||||
type cachedStmt struct {
|
||||
sql string
|
||||
args []interface{}
|
||||
}
|
||||
|
||||
func NewDelayedTx(db Database) *DelayedTx {
|
||||
return &DelayedTx{db: db}
|
||||
}
|
||||
|
||||
func (tx *DelayedTx) QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow {
|
||||
return tx.db.QueryRow(ctx, sql, args...)
|
||||
}
|
||||
|
||||
func (tx *DelayedTx) Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) {
|
||||
tx.cache = append(tx.cache, cachedStmt{sql, args})
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (tx *DelayedTx) Commit(ctx context.Context) error {
|
||||
base, err := tx.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
rollback(ctx, base)
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
rollback(ctx, base)
|
||||
}
|
||||
}()
|
||||
for _, stmt := range tx.cache {
|
||||
_, err := base.Exec(ctx, stmt.sql, stmt.args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
tx.cache = nil
|
||||
return base.Commit(ctx)
|
||||
}
|
||||
|
||||
func (tx *DelayedTx) Rollback(ctx context.Context) error {
|
||||
tx.cache = nil
|
||||
return nil
|
||||
}
|
||||
95
statediff/indexer/database/sql/mainnet_tests/indexer_test.go
Normal file
95
statediff/indexer/database/sql/mainnet_tests/indexer_test.go
Normal file
@ -0,0 +1,95 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package mainnet_tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
|
||||
)
|
||||
|
||||
var (
|
||||
err error
|
||||
db sql.Database
|
||||
ind interfaces.StateDiffIndexer
|
||||
chainConf = params.MainnetChainConfig
|
||||
)
|
||||
|
||||
func init() {
|
||||
if os.Getenv("MODE") != "statediff" {
|
||||
fmt.Println("Skipping statediff test")
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMainnetIndexer(t *testing.T) {
|
||||
conf := test_helpers.GetTestConfig()
|
||||
|
||||
for _, blockNumber := range test_helpers.ProblemBlocks {
|
||||
conf.BlockNumber = big.NewInt(blockNumber)
|
||||
tb, trs, err := test_helpers.TestBlockAndReceipts(conf)
|
||||
require.NoError(t, err)
|
||||
|
||||
testPushBlockAndState(t, tb, trs)
|
||||
}
|
||||
|
||||
testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf)
|
||||
require.NoError(t, err)
|
||||
|
||||
testPushBlockAndState(t, testBlock, testReceipts)
|
||||
}
|
||||
|
||||
func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
|
||||
t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
|
||||
setupMainnetIndexer(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
defer tearDown(t)
|
||||
|
||||
test.TestBlock(t, ind, block, receipts)
|
||||
})
|
||||
}
|
||||
|
||||
func setupMainnetIndexer(t *testing.T) {
|
||||
db, err = postgres.SetupSQLXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, db)
|
||||
}
|
||||
|
||||
func checkTxClosure(t *testing.T, idle, inUse, open int64) {
|
||||
require.Equal(t, idle, db.Stats().Idle())
|
||||
require.Equal(t, inUse, db.Stats().InUse())
|
||||
require.Equal(t, open, db.Stats().Open())
|
||||
}
|
||||
|
||||
func tearDown(t *testing.T) {
|
||||
test_helpers.TearDownDB(t, db)
|
||||
require.NoError(t, ind.Close())
|
||||
}
|
||||
52
statediff/indexer/database/sql/pgx_indexer_legacy_test.go
Normal file
52
statediff/indexer/database/sql/pgx_indexer_legacy_test.go
Normal file
@ -0,0 +1,52 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package sql_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
)
|
||||
|
||||
func setupLegacyPGXIndexer(t *testing.T) {
|
||||
db, err = postgres.SetupPGXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func setupLegacyPGX(t *testing.T) {
|
||||
setupLegacyPGXIndexer(t)
|
||||
test.SetupLegacyTestData(t, ind)
|
||||
}
|
||||
|
||||
func TestLegacyPGXIndexer(t *testing.T) {
|
||||
t.Run("Publish and index header IPLDs", func(t *testing.T) {
|
||||
setupLegacyPGX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestLegacyIndexer(t, db)
|
||||
})
|
||||
}
|
||||
227
statediff/indexer/database/sql/pgx_indexer_test.go
Normal file
227
statediff/indexer/database/sql/pgx_indexer_test.go
Normal file
@ -0,0 +1,227 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package sql_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/big"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
)
|
||||
|
||||
func setupPGXIndexer(t *testing.T) {
|
||||
db, err = postgres.SetupPGXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func setupPGX(t *testing.T) {
|
||||
setupPGXIndexer(t)
|
||||
test.SetupTestData(t, ind)
|
||||
}
|
||||
|
||||
func setupPGXNonCanonical(t *testing.T) {
|
||||
setupPGXIndexer(t)
|
||||
test.SetupTestDataNonCanonical(t, ind)
|
||||
}
|
||||
|
||||
// Test indexer for a canonical block
|
||||
func TestPGXIndexer(t *testing.T) {
|
||||
t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
|
||||
setupPGX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexHeaderIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
|
||||
setupPGX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexTransactionIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
|
||||
setupPGX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexLogIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
|
||||
setupPGX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexReceiptIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
|
||||
setupPGX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexStateIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
|
||||
setupPGX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexStorageIPLDs(t, db)
|
||||
})
|
||||
}
|
||||
|
||||
// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1
|
||||
func TestPGXIndexerNonCanonical(t *testing.T) {
|
||||
t.Run("Publish and index header", func(t *testing.T) {
|
||||
setupPGXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexHeaderNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index transactions", func(t *testing.T) {
|
||||
setupPGXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index receipts", func(t *testing.T) {
|
||||
setupPGXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index logs", func(t *testing.T) {
|
||||
setupPGXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexLogsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index state nodes", func(t *testing.T) {
|
||||
setupPGXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexStateNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index storage nodes", func(t *testing.T) {
|
||||
setupPGXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
test.TestPublishAndIndexStorageNonCanonical(t, db)
|
||||
})
|
||||
}
|
||||
|
||||
func TestPGXWatchAddressMethods(t *testing.T) {
|
||||
setupPGXIndexer(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 1, 0, 1)
|
||||
|
||||
t.Run("Load watched addresses (empty table)", func(t *testing.T) {
|
||||
test.TestLoadEmptyWatchedAddresses(t, ind)
|
||||
})
|
||||
|
||||
t.Run("Insert watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetInsertWatchedAddressesArgs()
|
||||
err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestInsertWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
|
||||
args := mocks.GetInsertAlreadyWatchedAddressesArgs()
|
||||
err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestInsertAlreadyWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Remove watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetRemoveWatchedAddressesArgs()
|
||||
err = ind.RemoveWatchedAddresses(args)
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestRemoveWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
|
||||
args := mocks.GetRemoveNonWatchedAddressesArgs()
|
||||
err = ind.RemoveWatchedAddresses(args)
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestRemoveNonWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Set watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetSetWatchedAddressesArgs()
|
||||
err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestSetWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
|
||||
args := mocks.GetSetAlreadyWatchedAddressesArgs()
|
||||
err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestSetAlreadyWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Load watched addresses", func(t *testing.T) {
|
||||
test.TestLoadWatchedAddresses(t, ind)
|
||||
})
|
||||
|
||||
t.Run("Clear watched addresses", func(t *testing.T) {
|
||||
err = ind.ClearWatchedAddresses()
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestClearWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
|
||||
err = ind.ClearWatchedAddresses()
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestClearEmptyWatchedAddresses(t, db)
|
||||
})
|
||||
}
|
||||
102
statediff/indexer/database/sql/postgres/config.go
Normal file
102
statediff/indexer/database/sql/postgres/config.go
Normal file
@ -0,0 +1,102 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
)
|
||||
|
||||
// DriverType to explicitly type the kind of sql driver we are using
|
||||
type DriverType string
|
||||
|
||||
const (
|
||||
PGX DriverType = "PGX"
|
||||
SQLX DriverType = "SQLX"
|
||||
Unknown DriverType = "Unknown"
|
||||
)
|
||||
|
||||
// ResolveDriverType resolves a DriverType from a provided string
|
||||
func ResolveDriverType(str string) (DriverType, error) {
|
||||
switch strings.ToLower(str) {
|
||||
case "pgx", "pgxpool":
|
||||
return PGX, nil
|
||||
case "sqlx":
|
||||
return SQLX, nil
|
||||
default:
|
||||
return Unknown, fmt.Errorf("unrecognized driver type string: %s", str)
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultConfig are default parameters for connecting to a Postgres sql
|
||||
var DefaultConfig = Config{
|
||||
Hostname: "localhost",
|
||||
Port: 8077,
|
||||
DatabaseName: "vulcanize_testing",
|
||||
Username: "vdbm",
|
||||
Password: "password",
|
||||
}
|
||||
|
||||
// Config holds params for a Postgres db
|
||||
type Config struct {
|
||||
// conn string params
|
||||
Hostname string
|
||||
Port int
|
||||
DatabaseName string
|
||||
Username string
|
||||
Password string
|
||||
|
||||
// conn settings
|
||||
MaxConns int
|
||||
MaxIdle int
|
||||
MinConns int
|
||||
MaxConnIdleTime time.Duration
|
||||
MaxConnLifetime time.Duration
|
||||
ConnTimeout time.Duration
|
||||
LogStatements bool
|
||||
|
||||
// node info params
|
||||
ID string
|
||||
ClientName string
|
||||
|
||||
// driver type
|
||||
Driver DriverType
|
||||
|
||||
// toggle on/off upserts
|
||||
Upsert bool
|
||||
}
|
||||
|
||||
// Type satisfies interfaces.Config
|
||||
func (c Config) Type() shared.DBType {
|
||||
return shared.POSTGRES
|
||||
}
|
||||
|
||||
// DbConnectionString constructs and returns the connection string from the config
|
||||
func (c Config) DbConnectionString() string {
|
||||
if len(c.Username) > 0 && len(c.Password) > 0 {
|
||||
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
|
||||
c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName)
|
||||
}
|
||||
if len(c.Username) > 0 && len(c.Password) == 0 {
|
||||
return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
|
||||
c.Username, c.Hostname, c.Port, c.DatabaseName)
|
||||
}
|
||||
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
|
||||
}
|
||||
123
statediff/indexer/database/sql/postgres/database.go
Normal file
123
statediff/indexer/database/sql/postgres/database.go
Normal file
@ -0,0 +1,123 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres
|
||||
|
||||
import "github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
|
||||
var _ sql.Database = &DB{}
|
||||
|
||||
const (
|
||||
createNodeStm = `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (node_id) DO NOTHING`
|
||||
)
|
||||
|
||||
// NewPostgresDB returns a postgres.DB using the provided driver
|
||||
func NewPostgresDB(driver sql.Driver, upsert bool) *DB {
|
||||
return &DB{upsert, driver}
|
||||
}
|
||||
|
||||
// DB implements sql.Database using a configured driver and Postgres statement syntax
|
||||
type DB struct {
|
||||
upsert bool
|
||||
sql.Driver
|
||||
}
|
||||
|
||||
// InsertHeaderStm satisfies the sql.Statements interface
|
||||
// Stm == Statement
|
||||
func (db *DB) InsertHeaderStm() string {
|
||||
if db.upsert {
|
||||
return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
|
||||
ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)`
|
||||
}
|
||||
return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
|
||||
ON CONFLICT (block_hash, block_number) DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertUncleStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertUncleStm() string {
|
||||
return `INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (block_hash, block_number) DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertTxStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertTxStm() string {
|
||||
return `INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertAccessListElementStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertAccessListElementStm() string {
|
||||
return `INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (tx_id, index, block_number) DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertRctStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertRctStm() string {
|
||||
return `INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT (tx_id, header_id, block_number) DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertLogStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertLogStm() string {
|
||||
return `INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||||
ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertStateStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertStateStm() string {
|
||||
if db.upsert {
|
||||
return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (header_id, state_path, block_number) DO UPDATE SET (block_number, state_leaf_key, cid, node_type, diff, mh_key) = ($1, $3, $4, $6, $7, $8)`
|
||||
}
|
||||
return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (header_id, state_path, block_number) DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertAccountStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertAccountStm() string {
|
||||
return `INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (header_id, state_path, block_number) DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertStorageStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertStorageStm() string {
|
||||
if db.upsert {
|
||||
return `INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (header_id, state_path, storage_path, block_number) DO UPDATE SET (block_number, storage_leaf_key, cid, node_type, diff, mh_key) = ($1, $4, $5, $7, $8, $9)`
|
||||
}
|
||||
return `INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (header_id, state_path, storage_path, block_number) DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertIPLDStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertIPLDStm() string {
|
||||
return `INSERT INTO public.blocks (block_number, key, data) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertIPLDsStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertIPLDsStm() string {
|
||||
return `INSERT INTO public.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING`
|
||||
}
|
||||
|
||||
// InsertKnownGapsStm satisfies the sql.Statements interface
|
||||
func (db *DB) InsertKnownGapsStm() string {
|
||||
return `INSERT INTO eth_meta.known_gaps (starting_block_number, ending_block_number, checked_out, processing_key) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (starting_block_number) DO UPDATE SET (ending_block_number, processing_key) = ($2, $4)
|
||||
WHERE eth_meta.known_gaps.ending_block_number <= $2`
|
||||
}
|
||||
38
statediff/indexer/database/sql/postgres/errors.go
Normal file
38
statediff/indexer/database/sql/postgres/errors.go
Normal file
@ -0,0 +1,38 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
DbConnectionFailedMsg = "db connection failed"
|
||||
SettingNodeFailedMsg = "unable to set db node"
|
||||
)
|
||||
|
||||
func ErrDBConnectionFailed(connectErr error) error {
|
||||
return formatError(DbConnectionFailedMsg, connectErr.Error())
|
||||
}
|
||||
|
||||
func ErrUnableToSetNode(setErr error) error {
|
||||
return formatError(SettingNodeFailedMsg, setErr.Error())
|
||||
}
|
||||
|
||||
func formatError(msg, err string) error {
|
||||
return fmt.Errorf("%s: %s", msg, err)
|
||||
}
|
||||
61
statediff/indexer/database/sql/postgres/log_adapter.go
Normal file
61
statediff/indexer/database/sql/postgres/log_adapter.go
Normal file
@ -0,0 +1,61 @@
|
||||
// Copyright © 2023 Cerc
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/jackc/pgx/v4"
|
||||
)
|
||||
|
||||
type LogAdapter struct {
|
||||
l log.Logger
|
||||
}
|
||||
|
||||
func NewLogAdapter(l log.Logger) *LogAdapter {
|
||||
return &LogAdapter{l: l}
|
||||
}
|
||||
|
||||
func (l *LogAdapter) Log(ctx context.Context, level pgx.LogLevel, msg string, data map[string]interface{}) {
|
||||
var logger log.Logger
|
||||
if data != nil {
|
||||
var args = make([]interface{}, 0)
|
||||
for key, value := range data {
|
||||
if value != nil {
|
||||
args = append(args, key, value)
|
||||
}
|
||||
}
|
||||
logger = l.l.New(args...)
|
||||
} else {
|
||||
logger = l.l
|
||||
}
|
||||
|
||||
switch level {
|
||||
case pgx.LogLevelTrace:
|
||||
logger.Trace(msg)
|
||||
case pgx.LogLevelDebug:
|
||||
logger.Debug(msg)
|
||||
case pgx.LogLevelInfo:
|
||||
logger.Info(msg)
|
||||
case pgx.LogLevelWarn:
|
||||
logger.Warn(msg)
|
||||
case pgx.LogLevelError:
|
||||
logger.Error(msg)
|
||||
default:
|
||||
logger.New("INVALID_PGX_LOG_LEVEL", level).Error(msg)
|
||||
}
|
||||
}
|
||||
241
statediff/indexer/database/sql/postgres/pgx.go
Normal file
241
statediff/indexer/database/sql/postgres/pgx.go
Normal file
@ -0,0 +1,241 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
|
||||
"github.com/georgysavva/scany/pgxscan"
|
||||
"github.com/jackc/pgconn"
|
||||
"github.com/jackc/pgx/v4"
|
||||
"github.com/jackc/pgx/v4/pgxpool"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
)
|
||||
|
||||
// PGXDriver driver, implements sql.Driver
|
||||
type PGXDriver struct {
|
||||
ctx context.Context
|
||||
pool *pgxpool.Pool
|
||||
nodeInfo node.Info
|
||||
nodeID string
|
||||
}
|
||||
|
||||
// NewPGXDriver returns a new pgx driver
|
||||
// it initializes the connection pool and creates the node info table
|
||||
func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) {
|
||||
pgConf, err := MakeConfig(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dbPool, err := pgxpool.ConnectConfig(ctx, pgConf)
|
||||
if err != nil {
|
||||
return nil, ErrDBConnectionFailed(err)
|
||||
}
|
||||
pg := &PGXDriver{ctx: ctx, pool: dbPool, nodeInfo: node}
|
||||
nodeErr := pg.createNode()
|
||||
if nodeErr != nil {
|
||||
return &PGXDriver{}, ErrUnableToSetNode(nodeErr)
|
||||
}
|
||||
return pg, nil
|
||||
}
|
||||
|
||||
// MakeConfig creates a pgxpool.Config from the provided Config
|
||||
func MakeConfig(config Config) (*pgxpool.Config, error) {
|
||||
conf, err := pgxpool.ParseConfig("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
//conf.ConnConfig.BuildStatementCache = nil
|
||||
conf.ConnConfig.Config.Host = config.Hostname
|
||||
conf.ConnConfig.Config.Port = uint16(config.Port)
|
||||
conf.ConnConfig.Config.Database = config.DatabaseName
|
||||
conf.ConnConfig.Config.User = config.Username
|
||||
conf.ConnConfig.Config.Password = config.Password
|
||||
|
||||
if config.ConnTimeout != 0 {
|
||||
conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout
|
||||
}
|
||||
if config.MaxConns != 0 {
|
||||
conf.MaxConns = int32(config.MaxConns)
|
||||
}
|
||||
if config.MinConns != 0 {
|
||||
conf.MinConns = int32(config.MinConns)
|
||||
}
|
||||
if config.MaxConnLifetime != 0 {
|
||||
conf.MaxConnLifetime = config.MaxConnLifetime
|
||||
}
|
||||
if config.MaxConnIdleTime != 0 {
|
||||
conf.MaxConnIdleTime = config.MaxConnIdleTime
|
||||
}
|
||||
|
||||
if config.LogStatements {
|
||||
conf.ConnConfig.Logger = NewLogAdapter(log.New())
|
||||
}
|
||||
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
func (pgx *PGXDriver) createNode() error {
|
||||
_, err := pgx.pool.Exec(
|
||||
pgx.ctx,
|
||||
createNodeStm,
|
||||
pgx.nodeInfo.GenesisBlock, pgx.nodeInfo.NetworkID,
|
||||
pgx.nodeInfo.ID, pgx.nodeInfo.ClientName,
|
||||
pgx.nodeInfo.ChainID)
|
||||
if err != nil {
|
||||
return ErrUnableToSetNode(err)
|
||||
}
|
||||
pgx.nodeID = pgx.nodeInfo.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
// QueryRow satisfies sql.Database
|
||||
func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
|
||||
return pgx.pool.QueryRow(ctx, sql, args...)
|
||||
}
|
||||
|
||||
// Exec satisfies sql.Database
|
||||
func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
|
||||
res, err := pgx.pool.Exec(ctx, sql, args...)
|
||||
return resultWrapper{ct: res}, err
|
||||
}
|
||||
|
||||
// Select satisfies sql.Database
|
||||
func (pgx *PGXDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
|
||||
return pgxscan.Select(ctx, pgx.pool, dest, query, args...)
|
||||
}
|
||||
|
||||
// Get satisfies sql.Database
|
||||
func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
|
||||
return pgxscan.Get(ctx, pgx.pool, dest, query, args...)
|
||||
}
|
||||
|
||||
// Begin satisfies sql.Database
|
||||
func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) {
|
||||
tx, err := pgx.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pgxTxWrapper{tx: tx}, nil
|
||||
}
|
||||
|
||||
func (pgx *PGXDriver) Stats() metrics.DbStats {
|
||||
stats := pgx.pool.Stat()
|
||||
return pgxStatsWrapper{stats: stats}
|
||||
}
|
||||
|
||||
// NodeID satisfies sql.Database
|
||||
func (pgx *PGXDriver) NodeID() string {
|
||||
return pgx.nodeID
|
||||
}
|
||||
|
||||
// Close satisfies sql.Database/io.Closer
|
||||
func (pgx *PGXDriver) Close() error {
|
||||
pgx.pool.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Context satisfies sql.Database
|
||||
func (pgx *PGXDriver) Context() context.Context {
|
||||
return pgx.ctx
|
||||
}
|
||||
|
||||
type resultWrapper struct {
|
||||
ct pgconn.CommandTag
|
||||
}
|
||||
|
||||
// RowsAffected satisfies sql.Result
|
||||
func (r resultWrapper) RowsAffected() (int64, error) {
|
||||
return r.ct.RowsAffected(), nil
|
||||
}
|
||||
|
||||
type pgxStatsWrapper struct {
|
||||
stats *pgxpool.Stat
|
||||
}
|
||||
|
||||
// MaxOpen satisfies metrics.DbStats
|
||||
func (s pgxStatsWrapper) MaxOpen() int64 {
|
||||
return int64(s.stats.MaxConns())
|
||||
}
|
||||
|
||||
// Open satisfies metrics.DbStats
|
||||
func (s pgxStatsWrapper) Open() int64 {
|
||||
return int64(s.stats.TotalConns())
|
||||
}
|
||||
|
||||
// InUse satisfies metrics.DbStats
|
||||
func (s pgxStatsWrapper) InUse() int64 {
|
||||
return int64(s.stats.AcquiredConns())
|
||||
}
|
||||
|
||||
// Idle satisfies metrics.DbStats
|
||||
func (s pgxStatsWrapper) Idle() int64 {
|
||||
return int64(s.stats.IdleConns())
|
||||
}
|
||||
|
||||
// WaitCount satisfies metrics.DbStats
|
||||
func (s pgxStatsWrapper) WaitCount() int64 {
|
||||
return s.stats.EmptyAcquireCount()
|
||||
}
|
||||
|
||||
// WaitDuration satisfies metrics.DbStats
|
||||
func (s pgxStatsWrapper) WaitDuration() time.Duration {
|
||||
return s.stats.AcquireDuration()
|
||||
}
|
||||
|
||||
// MaxIdleClosed satisfies metrics.DbStats
|
||||
func (s pgxStatsWrapper) MaxIdleClosed() int64 {
|
||||
// this stat isn't supported by pgxpool, but we don't want to panic
|
||||
return 0
|
||||
}
|
||||
|
||||
// MaxLifetimeClosed satisfies metrics.DbStats
|
||||
func (s pgxStatsWrapper) MaxLifetimeClosed() int64 {
|
||||
return s.stats.CanceledAcquireCount()
|
||||
}
|
||||
|
||||
type pgxTxWrapper struct {
|
||||
tx pgx.Tx
|
||||
}
|
||||
|
||||
// QueryRow satisfies sql.Tx
|
||||
func (t pgxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
|
||||
return t.tx.QueryRow(ctx, sql, args...)
|
||||
}
|
||||
|
||||
// Exec satisfies sql.Tx
|
||||
func (t pgxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
|
||||
res, err := t.tx.Exec(ctx, sql, args...)
|
||||
return resultWrapper{ct: res}, err
|
||||
}
|
||||
|
||||
// Commit satisfies sql.Tx
|
||||
func (t pgxTxWrapper) Commit(ctx context.Context) error {
|
||||
return t.tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// Rollback satisfies sql.Tx
|
||||
func (t pgxTxWrapper) Rollback(ctx context.Context) error {
|
||||
return t.tx.Rollback(ctx)
|
||||
}
|
||||
121
statediff/indexer/database/sql/postgres/pgx_test.go
Normal file
121
statediff/indexer/database/sql/postgres/pgx_test.go
Normal file
@ -0,0 +1,121 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v4/pgxpool"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
)
|
||||
|
||||
var (
|
||||
pgConfig, _ = postgres.MakeConfig(postgres.DefaultConfig)
|
||||
ctx = context.Background()
|
||||
)
|
||||
|
||||
func expectContainsSubstring(t *testing.T, full string, sub string) {
|
||||
if !strings.Contains(full, sub) {
|
||||
t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresPGX(t *testing.T) {
|
||||
t.Run("connects to the sql", func(t *testing.T) {
|
||||
dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err)
|
||||
}
|
||||
if dbPool == nil {
|
||||
t.Fatal("DB pool is nil")
|
||||
}
|
||||
dbPool.Close()
|
||||
})
|
||||
|
||||
t.Run("serializes big.Int to db", func(t *testing.T) {
|
||||
// postgres driver doesn't support go big.Int type
|
||||
// various casts in golang uint64, int64, overflow for
|
||||
// transaction value (in wei) even though
|
||||
// postgres numeric can handle an arbitrary
|
||||
// sized int, so use string representation of big.Int
|
||||
// and cast on insert
|
||||
|
||||
dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err)
|
||||
}
|
||||
defer dbPool.Close()
|
||||
|
||||
bi := new(big.Int)
|
||||
bi.SetString("34940183920000000000", 10)
|
||||
require.Equal(t, "34940183920000000000", bi.String())
|
||||
|
||||
defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`)
|
||||
_, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sqlStatement := `
|
||||
INSERT INTO example (id, data)
|
||||
VALUES (1, cast($1 AS NUMERIC))`
|
||||
_, err = dbPool.Exec(ctx, sqlStatement, bi.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var data string
|
||||
err = dbPool.QueryRow(ctx, `SELECT cast(data AS TEXT) FROM example WHERE id = 1`).Scan(&data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
require.Equal(t, data, bi.String())
|
||||
actual := new(big.Int)
|
||||
actual.SetString(data, 10)
|
||||
require.Equal(t, bi, actual)
|
||||
})
|
||||
|
||||
t.Run("throws error when can't connect to the database", func(t *testing.T) {
|
||||
goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
|
||||
_, err := postgres.NewPGXDriver(ctx, postgres.Config{}, goodInfo)
|
||||
if err == nil {
|
||||
t.Fatal("Expected an error")
|
||||
}
|
||||
|
||||
expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
|
||||
})
|
||||
|
||||
t.Run("throws error when can't create node", func(t *testing.T) {
|
||||
badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
|
||||
badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
|
||||
|
||||
_, err := postgres.NewPGXDriver(ctx, postgres.DefaultConfig, badInfo)
|
||||
if err == nil {
|
||||
t.Fatal("Expected an error")
|
||||
}
|
||||
|
||||
expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if os.Getenv("MODE") != "statediff" {
|
||||
fmt.Println("Skipping statediff test")
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
log.Root().SetHandler(log.DiscardHandler())
|
||||
}
|
||||
188
statediff/indexer/database/sql/postgres/sqlx.go
Normal file
188
statediff/indexer/database/sql/postgres/sqlx.go
Normal file
@ -0,0 +1,188 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
coresql "database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
)
|
||||
|
||||
// SQLXDriver driver, implements sql.Driver
|
||||
type SQLXDriver struct {
|
||||
ctx context.Context
|
||||
db *sqlx.DB
|
||||
nodeInfo node.Info
|
||||
nodeID string
|
||||
}
|
||||
|
||||
// NewSQLXDriver returns a new sqlx driver for Postgres
|
||||
// it initializes the connection pool and creates the node info table
|
||||
func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) {
|
||||
db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString())
|
||||
if err != nil {
|
||||
return &SQLXDriver{}, ErrDBConnectionFailed(err)
|
||||
}
|
||||
if config.MaxConns > 0 {
|
||||
db.SetMaxOpenConns(config.MaxConns)
|
||||
}
|
||||
if config.MaxConnLifetime > 0 {
|
||||
db.SetConnMaxLifetime(config.MaxConnLifetime)
|
||||
}
|
||||
db.SetMaxIdleConns(config.MaxIdle)
|
||||
driver := &SQLXDriver{ctx: ctx, db: db, nodeInfo: node}
|
||||
if err := driver.createNode(); err != nil {
|
||||
return &SQLXDriver{}, ErrUnableToSetNode(err)
|
||||
}
|
||||
return driver, nil
|
||||
}
|
||||
|
||||
func (driver *SQLXDriver) createNode() error {
|
||||
_, err := driver.db.Exec(
|
||||
createNodeStm,
|
||||
driver.nodeInfo.GenesisBlock, driver.nodeInfo.NetworkID,
|
||||
driver.nodeInfo.ID, driver.nodeInfo.ClientName,
|
||||
driver.nodeInfo.ChainID)
|
||||
if err != nil {
|
||||
return ErrUnableToSetNode(err)
|
||||
}
|
||||
driver.nodeID = driver.nodeInfo.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
// QueryRow satisfies sql.Database
|
||||
func (driver *SQLXDriver) QueryRow(_ context.Context, sql string, args ...interface{}) sql.ScannableRow {
|
||||
return driver.db.QueryRowx(sql, args...)
|
||||
}
|
||||
|
||||
// Exec satisfies sql.Database
|
||||
func (driver *SQLXDriver) Exec(_ context.Context, sql string, args ...interface{}) (sql.Result, error) {
|
||||
return driver.db.Exec(sql, args...)
|
||||
}
|
||||
|
||||
// Select satisfies sql.Database
|
||||
func (driver *SQLXDriver) Select(_ context.Context, dest interface{}, query string, args ...interface{}) error {
|
||||
return driver.db.Select(dest, query, args...)
|
||||
}
|
||||
|
||||
// Get satisfies sql.Database
|
||||
func (driver *SQLXDriver) Get(_ context.Context, dest interface{}, query string, args ...interface{}) error {
|
||||
return driver.db.Get(dest, query, args...)
|
||||
}
|
||||
|
||||
// Begin satisfies sql.Database
|
||||
func (driver *SQLXDriver) Begin(_ context.Context) (sql.Tx, error) {
|
||||
tx, err := driver.db.Beginx()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sqlxTxWrapper{tx: tx}, nil
|
||||
}
|
||||
|
||||
func (driver *SQLXDriver) Stats() metrics.DbStats {
|
||||
stats := driver.db.Stats()
|
||||
return sqlxStatsWrapper{stats: stats}
|
||||
}
|
||||
|
||||
// NodeID satisfies sql.Database
|
||||
func (driver *SQLXDriver) NodeID() string {
|
||||
return driver.nodeID
|
||||
}
|
||||
|
||||
// Close satisfies sql.Database/io.Closer
|
||||
func (driver *SQLXDriver) Close() error {
|
||||
return driver.db.Close()
|
||||
}
|
||||
|
||||
// Context satisfies sql.Database
|
||||
func (driver *SQLXDriver) Context() context.Context {
|
||||
return driver.ctx
|
||||
}
|
||||
|
||||
type sqlxStatsWrapper struct {
|
||||
stats coresql.DBStats
|
||||
}
|
||||
|
||||
// MaxOpen satisfies metrics.DbStats
|
||||
func (s sqlxStatsWrapper) MaxOpen() int64 {
|
||||
return int64(s.stats.MaxOpenConnections)
|
||||
}
|
||||
|
||||
// Open satisfies metrics.DbStats
|
||||
func (s sqlxStatsWrapper) Open() int64 {
|
||||
return int64(s.stats.OpenConnections)
|
||||
}
|
||||
|
||||
// InUse satisfies metrics.DbStats
|
||||
func (s sqlxStatsWrapper) InUse() int64 {
|
||||
return int64(s.stats.InUse)
|
||||
}
|
||||
|
||||
// Idle satisfies metrics.DbStats
|
||||
func (s sqlxStatsWrapper) Idle() int64 {
|
||||
return int64(s.stats.Idle)
|
||||
}
|
||||
|
||||
// WaitCount satisfies metrics.DbStats
|
||||
func (s sqlxStatsWrapper) WaitCount() int64 {
|
||||
return s.stats.WaitCount
|
||||
}
|
||||
|
||||
// WaitDuration satisfies metrics.DbStats
|
||||
func (s sqlxStatsWrapper) WaitDuration() time.Duration {
|
||||
return s.stats.WaitDuration
|
||||
}
|
||||
|
||||
// MaxIdleClosed satisfies metrics.DbStats
|
||||
func (s sqlxStatsWrapper) MaxIdleClosed() int64 {
|
||||
return s.stats.MaxIdleClosed
|
||||
}
|
||||
|
||||
// MaxLifetimeClosed satisfies metrics.DbStats
|
||||
func (s sqlxStatsWrapper) MaxLifetimeClosed() int64 {
|
||||
return s.stats.MaxLifetimeClosed
|
||||
}
|
||||
|
||||
type sqlxTxWrapper struct {
|
||||
tx *sqlx.Tx
|
||||
}
|
||||
|
||||
// QueryRow satisfies sql.Tx
|
||||
func (t sqlxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
|
||||
return t.tx.QueryRowx(sql, args...)
|
||||
}
|
||||
|
||||
// Exec satisfies sql.Tx
|
||||
func (t sqlxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
|
||||
return t.tx.Exec(sql, args...)
|
||||
}
|
||||
|
||||
// Commit satisfies sql.Tx
|
||||
func (t sqlxTxWrapper) Commit(ctx context.Context) error {
|
||||
return t.tx.Commit()
|
||||
}
|
||||
|
||||
// Rollback satisfies sql.Tx
|
||||
func (t sqlxTxWrapper) Rollback(ctx context.Context) error {
|
||||
return t.tx.Rollback()
|
||||
}
|
||||
119
statediff/indexer/database/sql/postgres/sqlx_test.go
Normal file
119
statediff/indexer/database/sql/postgres/sqlx_test.go
Normal file
@ -0,0 +1,119 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/big"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
)
|
||||
|
||||
func TestPostgresSQLX(t *testing.T) {
|
||||
var sqlxdb *sqlx.DB
|
||||
|
||||
t.Run("connects to the database", func(t *testing.T) {
|
||||
var err error
|
||||
connStr := postgres.DefaultConfig.DbConnectionString()
|
||||
|
||||
sqlxdb, err = sqlx.Connect("postgres", connStr)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err)
|
||||
}
|
||||
if sqlxdb == nil {
|
||||
t.Fatal("DB is nil")
|
||||
}
|
||||
err = sqlxdb.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("serializes big.Int to db", func(t *testing.T) {
|
||||
// postgres driver doesn't support go big.Int type
|
||||
// various casts in golang uint64, int64, overflow for
|
||||
// transaction value (in wei) even though
|
||||
// postgres numeric can handle an arbitrary
|
||||
// sized int, so use string representation of big.Int
|
||||
// and cast on insert
|
||||
|
||||
connStr := postgres.DefaultConfig.DbConnectionString()
|
||||
db, err := sqlx.Connect("postgres", connStr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
bi := new(big.Int)
|
||||
bi.SetString("34940183920000000000", 10)
|
||||
require.Equal(t, "34940183920000000000", bi.String())
|
||||
|
||||
defer db.Exec(`DROP TABLE IF EXISTS example`)
|
||||
_, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sqlStatement := `
|
||||
INSERT INTO example (id, data)
|
||||
VALUES (1, cast($1 AS NUMERIC))`
|
||||
_, err = db.Exec(sqlStatement, bi.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var data string
|
||||
err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
require.Equal(t, data, bi.String())
|
||||
actual := new(big.Int)
|
||||
actual.SetString(data, 10)
|
||||
require.Equal(t, bi, actual)
|
||||
})
|
||||
|
||||
t.Run("throws error when can't connect to the database", func(t *testing.T) {
|
||||
goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
|
||||
_, err := postgres.NewSQLXDriver(ctx, postgres.Config{}, goodInfo)
|
||||
if err == nil {
|
||||
t.Fatal("Expected an error")
|
||||
}
|
||||
|
||||
expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
|
||||
})
|
||||
|
||||
t.Run("throws error when can't create node", func(t *testing.T) {
|
||||
badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
|
||||
badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
|
||||
|
||||
_, err := postgres.NewSQLXDriver(ctx, postgres.DefaultConfig, badInfo)
|
||||
if err == nil {
|
||||
t.Fatal("Expected an error")
|
||||
}
|
||||
|
||||
expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
|
||||
})
|
||||
}
|
||||
44
statediff/indexer/database/sql/postgres/test_helpers.go
Normal file
44
statediff/indexer/database/sql/postgres/test_helpers.go
Normal file
@ -0,0 +1,44 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/node"
|
||||
)
|
||||
|
||||
// SetupSQLXDB is used to setup a sqlx db for tests
|
||||
func SetupSQLXDB() (sql.Database, error) {
|
||||
conf := DefaultConfig
|
||||
conf.MaxIdle = 0
|
||||
driver, err := NewSQLXDriver(context.Background(), conf, node.Info{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewPostgresDB(driver, false), nil
|
||||
}
|
||||
|
||||
// SetupPGXDB is used to setup a pgx db for tests
|
||||
func SetupPGXDB() (sql.Database, error) {
|
||||
driver, err := NewPGXDriver(context.Background(), DefaultConfig, node.Info{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewPostgresDB(driver, false), nil
|
||||
}
|
||||
52
statediff/indexer/database/sql/sqlx_indexer_legacy_test.go
Normal file
52
statediff/indexer/database/sql/sqlx_indexer_legacy_test.go
Normal file
@ -0,0 +1,52 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package sql_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
)
|
||||
|
||||
func setupLegacySQLXIndexer(t *testing.T) {
|
||||
db, err = postgres.SetupSQLXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func setupLegacySQLX(t *testing.T) {
|
||||
setupLegacySQLXIndexer(t)
|
||||
test.SetupLegacyTestData(t, ind)
|
||||
}
|
||||
|
||||
func TestLegacySQLXIndexer(t *testing.T) {
|
||||
t.Run("Publish and index header IPLDs", func(t *testing.T) {
|
||||
setupLegacySQLX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestLegacyIndexer(t, db)
|
||||
})
|
||||
}
|
||||
227
statediff/indexer/database/sql/sqlx_indexer_test.go
Normal file
227
statediff/indexer/database/sql/sqlx_indexer_test.go
Normal file
@ -0,0 +1,227 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package sql_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/big"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/test"
|
||||
)
|
||||
|
||||
func setupSQLXIndexer(t *testing.T) {
|
||||
db, err = postgres.SetupSQLXDB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, db)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func setupSQLX(t *testing.T) {
|
||||
setupSQLXIndexer(t)
|
||||
test.SetupTestData(t, ind)
|
||||
}
|
||||
|
||||
func setupSQLXNonCanonical(t *testing.T) {
|
||||
setupSQLXIndexer(t)
|
||||
test.SetupTestDataNonCanonical(t, ind)
|
||||
}
|
||||
|
||||
// Test indexer for a canonical block
|
||||
func TestSQLXIndexer(t *testing.T) {
|
||||
t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
|
||||
setupSQLX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexHeaderIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
|
||||
setupSQLX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexTransactionIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
|
||||
setupSQLX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexLogIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
|
||||
setupSQLX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexReceiptIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
|
||||
setupSQLX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexStateIPLDs(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
|
||||
setupSQLX(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexStorageIPLDs(t, db)
|
||||
})
|
||||
}
|
||||
|
||||
// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1
|
||||
func TestSQLXIndexerNonCanonical(t *testing.T) {
|
||||
t.Run("Publish and index header", func(t *testing.T) {
|
||||
setupSQLXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexHeaderNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index transactions", func(t *testing.T) {
|
||||
setupSQLXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index receipts", func(t *testing.T) {
|
||||
setupSQLXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index logs", func(t *testing.T) {
|
||||
setupSQLXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexLogsNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index state nodes", func(t *testing.T) {
|
||||
setupSQLXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexStateNonCanonical(t, db)
|
||||
})
|
||||
|
||||
t.Run("Publish and index storage nodes", func(t *testing.T) {
|
||||
setupSQLXNonCanonical(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
test.TestPublishAndIndexStorageNonCanonical(t, db)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSQLXWatchAddressMethods(t *testing.T) {
|
||||
setupSQLXIndexer(t)
|
||||
defer tearDown(t)
|
||||
defer checkTxClosure(t, 0, 0, 0)
|
||||
|
||||
t.Run("Load watched addresses (empty table)", func(t *testing.T) {
|
||||
test.TestLoadEmptyWatchedAddresses(t, ind)
|
||||
})
|
||||
|
||||
t.Run("Insert watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetInsertWatchedAddressesArgs()
|
||||
err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt1)))
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestInsertWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Insert watched addresses (some already watched)", func(t *testing.T) {
|
||||
args := mocks.GetInsertAlreadyWatchedAddressesArgs()
|
||||
err = ind.InsertWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestInsertAlreadyWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Remove watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetRemoveWatchedAddressesArgs()
|
||||
err = ind.RemoveWatchedAddresses(args)
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestRemoveWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Remove watched addresses (some non-watched)", func(t *testing.T) {
|
||||
args := mocks.GetRemoveNonWatchedAddressesArgs()
|
||||
err = ind.RemoveWatchedAddresses(args)
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestRemoveNonWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Set watched addresses", func(t *testing.T) {
|
||||
args := mocks.GetSetWatchedAddressesArgs()
|
||||
err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt2)))
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestSetWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Set watched addresses (some already watched)", func(t *testing.T) {
|
||||
args := mocks.GetSetAlreadyWatchedAddressesArgs()
|
||||
err = ind.SetWatchedAddresses(args, big.NewInt(int64(mocks.WatchedAt3)))
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestSetAlreadyWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Load watched addresses", func(t *testing.T) {
|
||||
test.TestLoadWatchedAddresses(t, ind)
|
||||
})
|
||||
|
||||
t.Run("Clear watched addresses", func(t *testing.T) {
|
||||
err = ind.ClearWatchedAddresses()
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestClearWatchedAddresses(t, db)
|
||||
})
|
||||
|
||||
t.Run("Clear watched addresses (empty table)", func(t *testing.T) {
|
||||
err = ind.ClearWatchedAddresses()
|
||||
require.NoError(t, err)
|
||||
|
||||
test.TestClearEmptyWatchedAddresses(t, db)
|
||||
})
|
||||
}
|
||||
203
statediff/indexer/database/sql/writer.go
Normal file
203
statediff/indexer/database/sql/writer.go
Normal file
@ -0,0 +1,203 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package sql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/models"
|
||||
)
|
||||
|
||||
var (
|
||||
nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
|
||||
)
|
||||
|
||||
// Writer handles processing and writing of indexed IPLD objects to Postgres
|
||||
type Writer struct {
|
||||
db Database
|
||||
}
|
||||
|
||||
// NewWriter creates a new pointer to a Writer
|
||||
func NewWriter(db Database) *Writer {
|
||||
return &Writer{
|
||||
db: db,
|
||||
}
|
||||
}
|
||||
|
||||
// Close satisfies io.Closer
|
||||
func (w *Writer) Close() error {
|
||||
return w.db.Close()
|
||||
}
|
||||
|
||||
/*
|
||||
INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
|
||||
ON CONFLICT (block_hash, block_number) DO UPDATE SET (block_number, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($1, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)
|
||||
*/
|
||||
func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error {
|
||||
_, err := tx.Exec(w.db.Context(), w.db.InsertHeaderStm(),
|
||||
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, w.db.NodeID(),
|
||||
header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UncleRoot, header.Bloom,
|
||||
header.Timestamp, header.MhKey, 1, header.Coinbase)
|
||||
if err != nil {
|
||||
return insertError{"eth.header_cids", err, w.db.InsertHeaderStm(), header}
|
||||
}
|
||||
metrics.IndexerMetrics.BlocksCounter.Inc(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (block_hash, block_number) DO NOTHING
|
||||
*/
|
||||
func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error {
|
||||
_, err := tx.Exec(w.db.Context(), w.db.InsertUncleStm(),
|
||||
uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.MhKey)
|
||||
if err != nil {
|
||||
return insertError{"eth.uncle_cids", err, w.db.InsertUncleStm(), uncle}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING
|
||||
*/
|
||||
func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
|
||||
_, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(),
|
||||
transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src,
|
||||
transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value)
|
||||
if err != nil {
|
||||
return insertError{"eth.transaction_cids", err, w.db.InsertTxStm(), transaction}
|
||||
}
|
||||
metrics.IndexerMetrics.TransactionsCounter.Inc(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (tx_id, index, block_number) DO NOTHING
|
||||
*/
|
||||
func (w *Writer) upsertAccessListElement(tx Tx, accessListElement models.AccessListElementModel) error {
|
||||
_, err := tx.Exec(w.db.Context(), w.db.InsertAccessListElementStm(),
|
||||
accessListElement.BlockNumber, accessListElement.TxID, accessListElement.Index, accessListElement.Address,
|
||||
accessListElement.StorageKeys)
|
||||
if err != nil {
|
||||
return insertError{"eth.access_list_elements", err, w.db.InsertAccessListElementStm(), accessListElement}
|
||||
}
|
||||
metrics.IndexerMetrics.AccessListEntriesCounter.Inc(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT (tx_id, header_id, block_number) DO NOTHING
|
||||
*/
|
||||
func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
|
||||
_, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(),
|
||||
rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, rct.PostState,
|
||||
rct.PostStatus, rct.LogRoot)
|
||||
if err != nil {
|
||||
return insertError{"eth.receipt_cids", err, w.db.InsertRctStm(), *rct}
|
||||
}
|
||||
metrics.IndexerMetrics.ReceiptsCounter.Inc(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||||
ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING
|
||||
*/
|
||||
func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
|
||||
for _, log := range logs {
|
||||
_, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(),
|
||||
log.BlockNumber, log.HeaderID, log.LeafCID, log.LeafMhKey, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1,
|
||||
log.Topic2, log.Topic3, log.Data)
|
||||
if err != nil {
|
||||
return insertError{"eth.log_cids", err, w.db.InsertLogStm(), *log}
|
||||
}
|
||||
metrics.IndexerMetrics.LogsCounter.Inc(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (header_id, state_path, block_number) DO UPDATE SET (block_number, state_leaf_key, cid, node_type, diff, mh_key) = ($1 $3, $4, $6, $7, $8)
|
||||
*/
|
||||
func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
|
||||
var stateKey string
|
||||
if stateNode.StateKey != nullHash.String() {
|
||||
stateKey = stateNode.StateKey
|
||||
}
|
||||
_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
|
||||
stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true,
|
||||
stateNode.MhKey)
|
||||
if err != nil {
|
||||
return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (header_id, state_path, block_number) DO NOTHING
|
||||
*/
|
||||
func (w *Writer) upsertStateAccount(tx Tx, stateAccount models.StateAccountModel) error {
|
||||
_, err := tx.Exec(w.db.Context(), w.db.InsertAccountStm(),
|
||||
stateAccount.BlockNumber, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance,
|
||||
stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot)
|
||||
if err != nil {
|
||||
return insertError{"eth.state_accounts", err, w.db.InsertAccountStm(), stateAccount}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (header_id, state_path, storage_path, block_number) DO UPDATE SET (block_number, storage_leaf_key, cid, node_type, diff, mh_key) = ($1, $4, $5, $7, $8, $9)
|
||||
*/
|
||||
func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error {
|
||||
var storageKey string
|
||||
if storageCID.StorageKey != nullHash.String() {
|
||||
storageKey = storageCID.StorageKey
|
||||
}
|
||||
_, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(),
|
||||
storageCID.BlockNumber, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID, storageCID.Path,
|
||||
storageCID.NodeType, true, storageCID.MhKey)
|
||||
if err != nil {
|
||||
return insertError{"eth.storage_cids", err, w.db.InsertStorageStm(), storageCID}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type insertError struct {
|
||||
table string
|
||||
err error
|
||||
stmt string
|
||||
arguments interface{}
|
||||
}
|
||||
|
||||
var _ error = insertError{}
|
||||
|
||||
func (dbe insertError) Error() string {
|
||||
return fmt.Sprintf("error inserting %s entry: %v\r\nstatement: %s\r\narguments: %+v",
|
||||
dbe.table, dbe.err, dbe.stmt, dbe.arguments)
|
||||
}
|
||||
55
statediff/indexer/interfaces/interfaces.go
Normal file
55
statediff/indexer/interfaces/interfaces.go
Normal file
@ -0,0 +1,55 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2021 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package interfaces
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/big"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
|
||||
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
|
||||
)
|
||||
|
||||
// StateDiffIndexer interface required to index statediff data
|
||||
type StateDiffIndexer interface {
|
||||
PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (Batch, error)
|
||||
PushStateNode(tx Batch, stateNode sdtypes.StateNode, headerID string) error
|
||||
PushCodeAndCodeHash(tx Batch, codeAndCodeHash sdtypes.CodeAndCodeHash) error
|
||||
ReportDBMetrics(delay time.Duration, quit <-chan bool)
|
||||
|
||||
// Methods used by WatchAddress API/functionality
|
||||
LoadWatchedAddresses() ([]common.Address, error)
|
||||
InsertWatchedAddresses(addresses []sdtypes.WatchAddressArg, currentBlock *big.Int) error
|
||||
RemoveWatchedAddresses(addresses []sdtypes.WatchAddressArg) error
|
||||
SetWatchedAddresses(args []sdtypes.WatchAddressArg, currentBlockNumber *big.Int) error
|
||||
ClearWatchedAddresses() error
|
||||
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Batch required for indexing data atomically
|
||||
type Batch interface {
|
||||
Submit(err error) error
|
||||
}
|
||||
|
||||
// Config used to configure different underlying implementations
|
||||
type Config interface {
|
||||
Type() shared.DBType
|
||||
}
|
||||
BIN
statediff/indexer/ipld/eip2930_test_data/eth-block-12252078
Normal file
BIN
statediff/indexer/ipld/eip2930_test_data/eth-block-12252078
Normal file
Binary file not shown.
BIN
statediff/indexer/ipld/eip2930_test_data/eth-block-12365585
Normal file
BIN
statediff/indexer/ipld/eip2930_test_data/eth-block-12365585
Normal file
Binary file not shown.
BIN
statediff/indexer/ipld/eip2930_test_data/eth-block-12365586
Normal file
BIN
statediff/indexer/ipld/eip2930_test_data/eth-block-12365586
Normal file
Binary file not shown.
BIN
statediff/indexer/ipld/eip2930_test_data/eth-receipts-12252078
Normal file
BIN
statediff/indexer/ipld/eip2930_test_data/eth-receipts-12252078
Normal file
Binary file not shown.
BIN
statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365585
Normal file
BIN
statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365585
Normal file
Binary file not shown.
BIN
statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365586
Normal file
BIN
statediff/indexer/ipld/eip2930_test_data/eth-receipts-12365586
Normal file
Binary file not shown.
175
statediff/indexer/ipld/eth_account.go
Normal file
175
statediff/indexer/ipld/eth_account.go
Normal file
@ -0,0 +1,175 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package ipld
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/big"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
)
|
||||
|
||||
// EthAccountSnapshot (eth-account-snapshot codec 0x97)
|
||||
// represents an ethereum account, i.e. a wallet address or
|
||||
// a smart contract
|
||||
type EthAccountSnapshot struct {
|
||||
*EthAccount
|
||||
|
||||
cid cid.Cid
|
||||
rawdata []byte
|
||||
}
|
||||
|
||||
// EthAccount is the building block of EthAccountSnapshot.
|
||||
// Or, is the former stripped of its cid and rawdata components.
|
||||
type EthAccount struct {
|
||||
Nonce uint64
|
||||
Balance *big.Int
|
||||
Root []byte // This is the storage root trie
|
||||
CodeHash []byte // This is the hash of the EVM code
|
||||
}
|
||||
|
||||
// Static (compile time) check that EthAccountSnapshot satisfies the
|
||||
// node.Node interface.
|
||||
var _ node.Node = (*EthAccountSnapshot)(nil)
|
||||
|
||||
/*
|
||||
INPUT
|
||||
*/
|
||||
|
||||
// Input should be managed by EthStateTrie
|
||||
|
||||
/*
|
||||
OUTPUT
|
||||
*/
|
||||
|
||||
// Output should be managed by EthStateTrie
|
||||
|
||||
/*
|
||||
Block INTERFACE
|
||||
*/
|
||||
|
||||
// RawData returns the binary of the RLP encode of the account snapshot.
|
||||
func (as *EthAccountSnapshot) RawData() []byte {
|
||||
return as.rawdata
|
||||
}
|
||||
|
||||
// Cid returns the cid of the transaction.
|
||||
func (as *EthAccountSnapshot) Cid() cid.Cid {
|
||||
return as.cid
|
||||
}
|
||||
|
||||
// String is a helper for output
|
||||
func (as *EthAccountSnapshot) String() string {
|
||||
return fmt.Sprintf("<EthereumAccountSnapshot %s>", as.cid)
|
||||
}
|
||||
|
||||
// Loggable returns in a map the type of IPLD Link.
|
||||
func (as *EthAccountSnapshot) Loggable() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "eth-account-snapshot",
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Node INTERFACE
|
||||
*/
|
||||
|
||||
// Resolve resolves a path through this node, stopping at any link boundary
|
||||
// and returning the object found as well as the remaining path to traverse
|
||||
func (as *EthAccountSnapshot) Resolve(p []string) (interface{}, []string, error) {
|
||||
if len(p) == 0 {
|
||||
return as, nil, nil
|
||||
}
|
||||
|
||||
if len(p) > 1 {
|
||||
return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
|
||||
}
|
||||
|
||||
switch p[0] {
|
||||
case "balance":
|
||||
return as.Balance, nil, nil
|
||||
case "codeHash":
|
||||
return &node.Link{Cid: keccak256ToCid(RawBinary, as.CodeHash)}, nil, nil
|
||||
case "nonce":
|
||||
return as.Nonce, nil, nil
|
||||
case "root":
|
||||
return &node.Link{Cid: keccak256ToCid(MEthStorageTrie, as.Root)}, nil, nil
|
||||
default:
|
||||
return nil, nil, ErrInvalidLink
|
||||
}
|
||||
}
|
||||
|
||||
// Tree lists all paths within the object under 'path', and up to the given depth.
|
||||
// To list the entire object (similar to `find .`) pass "" and -1
|
||||
func (as *EthAccountSnapshot) Tree(p string, depth int) []string {
|
||||
if p != "" || depth == 0 {
|
||||
return nil
|
||||
}
|
||||
return []string{"balance", "codeHash", "nonce", "root"}
|
||||
}
|
||||
|
||||
// ResolveLink is a helper function that calls resolve and asserts the
|
||||
// output is a link
|
||||
func (as *EthAccountSnapshot) ResolveLink(p []string) (*node.Link, []string, error) {
|
||||
obj, rest, err := as.Resolve(p)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if lnk, ok := obj.(*node.Link); ok {
|
||||
return lnk, rest, nil
|
||||
}
|
||||
|
||||
return nil, nil, fmt.Errorf("resolved item was not a link")
|
||||
}
|
||||
|
||||
// Copy will go away. It is here to comply with the interface.
|
||||
func (as *EthAccountSnapshot) Copy() node.Node {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
// Links is a helper function that returns all links within this object
|
||||
func (as *EthAccountSnapshot) Links() []*node.Link {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat will go away. It is here to comply with the interface.
|
||||
func (as *EthAccountSnapshot) Stat() (*node.NodeStat, error) {
|
||||
return &node.NodeStat{}, nil
|
||||
}
|
||||
|
||||
// Size will go away. It is here to comply with the interface.
|
||||
func (as *EthAccountSnapshot) Size() (uint64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
/*
|
||||
EthAccountSnapshot functions
|
||||
*/
|
||||
|
||||
// MarshalJSON processes the transaction into readable JSON format.
|
||||
func (as *EthAccountSnapshot) MarshalJSON() ([]byte, error) {
|
||||
out := map[string]interface{}{
|
||||
"balance": as.Balance,
|
||||
"codeHash": keccak256ToCid(RawBinary, as.CodeHash),
|
||||
"nonce": as.Nonce,
|
||||
"root": keccak256ToCid(MEthStorageTrie, as.Root),
|
||||
}
|
||||
return json.Marshal(out)
|
||||
}
|
||||
295
statediff/indexer/ipld/eth_account_test.go
Normal file
295
statediff/indexer/ipld/eth_account_test.go
Normal file
@ -0,0 +1,295 @@
|
||||
package ipld
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Block INTERFACE
|
||||
func init() {
|
||||
if os.Getenv("MODE") != "statediff" {
|
||||
fmt.Println("Skipping statediff test")
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountSnapshotBlockElements(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
if fmt.Sprintf("%x", eas.RawData())[:10] != "f84e808a03" {
|
||||
t.Fatal("Wrong Data")
|
||||
}
|
||||
|
||||
if eas.Cid().String() !=
|
||||
"baglqcgzasckx2alxk43cksshnztjvhfyvbbh6bkp376gtcndm5cg4fkrkhsa" {
|
||||
t.Fatal("Wrong Cid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountSnapshotString(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
if eas.String() !=
|
||||
"<EthereumAccountSnapshot baglqcgzasckx2alxk43cksshnztjvhfyvbbh6bkp376gtcndm5cg4fkrkhsa>" {
|
||||
t.Fatalf("Wrong String()")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountSnapshotLoggable(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
l := eas.Loggable()
|
||||
if _, ok := l["type"]; !ok {
|
||||
t.Fatal("Loggable map expected the field 'type'")
|
||||
}
|
||||
|
||||
if l["type"] != "eth-account-snapshot" {
|
||||
t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-account-snapshot", l["type"])
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Node INTERFACE
|
||||
*/
|
||||
func TestAccountSnapshotResolve(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
// Empty path
|
||||
obj, rest, err := eas.Resolve([]string{})
|
||||
reas, ok := obj.(*EthAccountSnapshot)
|
||||
if !ok {
|
||||
t.Fatalf("Wrong type of returned object\r\nexpected %T\r\ngot %T", &EthAccountSnapshot{}, reas)
|
||||
}
|
||||
if reas.Cid() != eas.Cid() {
|
||||
t.Fatalf("wrong returned CID\r\nexpected %s\r\ngot %s", eas.Cid().String(), reas.Cid().String())
|
||||
}
|
||||
if rest != nil {
|
||||
t.Fatal("rest should be nil")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal("err should be nil")
|
||||
}
|
||||
|
||||
// len(p) > 1
|
||||
badCases := [][]string{
|
||||
{"two", "elements"},
|
||||
{"here", "three", "elements"},
|
||||
{"and", "here", "four", "elements"},
|
||||
}
|
||||
|
||||
for _, bc := range badCases {
|
||||
obj, rest, err = eas.Resolve(bc)
|
||||
if obj != nil {
|
||||
t.Fatal("obj should be nil")
|
||||
}
|
||||
if rest != nil {
|
||||
t.Fatal("rest should be nil")
|
||||
}
|
||||
if err.Error() != fmt.Sprintf("unexpected path elements past %s", bc[0]) {
|
||||
t.Fatal("wrong error")
|
||||
}
|
||||
}
|
||||
|
||||
moreBadCases := []string{
|
||||
"i",
|
||||
"am",
|
||||
"not",
|
||||
"an",
|
||||
"account",
|
||||
"field",
|
||||
}
|
||||
for _, mbc := range moreBadCases {
|
||||
obj, rest, err = eas.Resolve([]string{mbc})
|
||||
if obj != nil {
|
||||
t.Fatal("obj should be nil")
|
||||
}
|
||||
if rest != nil {
|
||||
t.Fatal("rest should be nil")
|
||||
}
|
||||
if err != ErrInvalidLink {
|
||||
t.Fatal("wrong error")
|
||||
}
|
||||
}
|
||||
|
||||
goodCases := []string{
|
||||
"balance",
|
||||
"codeHash",
|
||||
"nonce",
|
||||
"root",
|
||||
}
|
||||
for _, gc := range goodCases {
|
||||
_, _, err = eas.Resolve([]string{gc})
|
||||
if err != nil {
|
||||
t.Fatalf("error should be nil %v", gc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountSnapshotTree(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
// Bad cases
|
||||
tree := eas.Tree("non-empty-string", 0)
|
||||
if tree != nil {
|
||||
t.Fatal("Expected nil to be returned")
|
||||
}
|
||||
|
||||
tree = eas.Tree("non-empty-string", 1)
|
||||
if tree != nil {
|
||||
t.Fatal("Expected nil to be returned")
|
||||
}
|
||||
|
||||
tree = eas.Tree("", 0)
|
||||
if tree != nil {
|
||||
t.Fatal("Expected nil to be returned")
|
||||
}
|
||||
|
||||
// Good cases
|
||||
tree = eas.Tree("", 1)
|
||||
lookupElements := map[string]interface{}{
|
||||
"balance": nil,
|
||||
"codeHash": nil,
|
||||
"nonce": nil,
|
||||
"root": nil,
|
||||
}
|
||||
|
||||
if len(tree) != len(lookupElements) {
|
||||
t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree))
|
||||
}
|
||||
|
||||
for _, te := range tree {
|
||||
if _, ok := lookupElements[te]; !ok {
|
||||
t.Fatalf("Unexpected Element: %v", te)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountSnapshotResolveLink(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
// bad case
|
||||
obj, rest, err := eas.ResolveLink([]string{"supercalifragilist"})
|
||||
if obj != nil {
|
||||
t.Fatalf("Expected obj to be nil")
|
||||
}
|
||||
if rest != nil {
|
||||
t.Fatal("Expected rest to be nil")
|
||||
}
|
||||
if err != ErrInvalidLink {
|
||||
t.Fatal("Wrong error")
|
||||
}
|
||||
|
||||
// good case
|
||||
obj, rest, err = eas.ResolveLink([]string{"nonce"})
|
||||
if obj != nil {
|
||||
t.Fatalf("Expected obj to be nil")
|
||||
}
|
||||
if rest != nil {
|
||||
t.Fatal("Expected rest to be nil")
|
||||
}
|
||||
if err.Error() != "resolved item was not a link" {
|
||||
t.Fatal("Wrong error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountSnapshotCopy(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
defer func() {
|
||||
r := recover()
|
||||
if r == nil {
|
||||
t.Fatal("Expected panic")
|
||||
}
|
||||
if r != "implement me" {
|
||||
t.Fatalf("Wrong panic message\r\n expected %s\r\ngot %s", "'implement me'", r)
|
||||
}
|
||||
}()
|
||||
|
||||
_ = eas.Copy()
|
||||
}
|
||||
|
||||
func TestAccountSnapshotLinks(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
if eas.Links() != nil {
|
||||
t.Fatal("Links() expected to return nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountSnapshotStat(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
obj, err := eas.Stat()
|
||||
if obj == nil {
|
||||
t.Fatal("Expected a not null object node.NodeStat")
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatal("Expected a nil error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountSnapshotSize(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
size, err := eas.Size()
|
||||
if size != uint64(0) {
|
||||
t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", 0, size)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatal("Expected a nil error")
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
EthAccountSnapshot functions
|
||||
*/
|
||||
|
||||
func TestAccountSnapshotMarshalJSON(t *testing.T) {
|
||||
eas := prepareEthAccountSnapshot(t)
|
||||
|
||||
jsonOutput, err := eas.MarshalJSON()
|
||||
checkError(err, t)
|
||||
|
||||
var data map[string]interface{}
|
||||
err = json.Unmarshal(jsonOutput, &data)
|
||||
checkError(err, t)
|
||||
|
||||
balanceExpression := regexp.MustCompile(`{"balance":16011846000000000000000,`)
|
||||
if !balanceExpression.MatchString(string(jsonOutput)) {
|
||||
t.Fatal("Balance expression not found")
|
||||
}
|
||||
|
||||
code, _ := data["codeHash"].(map[string]interface{})
|
||||
if fmt.Sprintf("%s", code["/"]) !=
|
||||
"bafkrwigf2jdadbxxem6je7t5wlomoa6a4ualmu6kqittw6723acf3bneoa" {
|
||||
t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "bafkrwigf2jdadbxxem6je7t5wlomoa6a4ualmu6kqittw6723acf3bneoa", fmt.Sprintf("%s", code["/"]))
|
||||
}
|
||||
|
||||
if fmt.Sprintf("%v", data["nonce"]) != "0" {
|
||||
t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "0", fmt.Sprintf("%v", data["nonce"]))
|
||||
}
|
||||
|
||||
root, _ := data["root"].(map[string]interface{})
|
||||
if fmt.Sprintf("%s", root["/"]) !=
|
||||
"bagmacgzak3ub6fy3zrk2n74dixtjfqhynznurya3tfwk3qabmix3ly3dwqqq" {
|
||||
t.Fatalf("Wrong Marshaled Value\r\nexpected %s\r\ngot %s", "bagmacgzak3ub6fy3zrk2n74dixtjfqhynznurya3tfwk3qabmix3ly3dwqqq", fmt.Sprintf("%s", root["/"]))
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
AUXILIARS
|
||||
*/
|
||||
func prepareEthAccountSnapshot(t *testing.T) *EthAccountSnapshot {
|
||||
fi, err := os.Open("test_data/eth-state-trie-rlp-c9070d")
|
||||
checkError(err, t)
|
||||
|
||||
output, err := FromStateTrieRLPFile(fi)
|
||||
checkError(err, t)
|
||||
|
||||
return output.elements[1].(*EthAccountSnapshot)
|
||||
}
|
||||
293
statediff/indexer/ipld/eth_header.go
Normal file
293
statediff/indexer/ipld/eth_header.go
Normal file
@ -0,0 +1,293 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package ipld
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
|
||||
// EthHeader (eth-block, codec 0x90), represents an ethereum block header
|
||||
type EthHeader struct {
|
||||
*types.Header
|
||||
|
||||
cid cid.Cid
|
||||
rawdata []byte
|
||||
}
|
||||
|
||||
// Static (compile time) check that EthHeader satisfies the node.Node interface.
|
||||
var _ node.Node = (*EthHeader)(nil)
|
||||
|
||||
/*
|
||||
INPUT
|
||||
*/
|
||||
|
||||
// NewEthHeader converts a *types.Header into an EthHeader IPLD node
|
||||
func NewEthHeader(header *types.Header) (*EthHeader, error) {
|
||||
headerRLP, err := rlp.EncodeToBytes(header)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := RawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &EthHeader{
|
||||
Header: header,
|
||||
cid: c,
|
||||
rawdata: headerRLP,
|
||||
}, nil
|
||||
}
|
||||
|
||||
/*
|
||||
OUTPUT
|
||||
*/
|
||||
|
||||
// DecodeEthHeader takes a cid and its raw binary data
|
||||
// from IPFS and returns an EthTx object for further processing.
|
||||
func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) {
|
||||
h := new(types.Header)
|
||||
if err := rlp.DecodeBytes(b, h); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &EthHeader{
|
||||
Header: h,
|
||||
cid: c,
|
||||
rawdata: b,
|
||||
}, nil
|
||||
}
|
||||
|
||||
/*
|
||||
Block INTERFACE
|
||||
*/
|
||||
|
||||
// RawData returns the binary of the RLP encode of the block header.
|
||||
func (b *EthHeader) RawData() []byte {
|
||||
return b.rawdata
|
||||
}
|
||||
|
||||
// Cid returns the cid of the block header.
|
||||
func (b *EthHeader) Cid() cid.Cid {
|
||||
return b.cid
|
||||
}
|
||||
|
||||
// String is a helper for output
|
||||
func (b *EthHeader) String() string {
|
||||
return fmt.Sprintf("<EthHeader %s>", b.cid)
|
||||
}
|
||||
|
||||
// Loggable returns a map the type of IPLD Link.
|
||||
func (b *EthHeader) Loggable() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "eth-header",
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Node INTERFACE
|
||||
*/
|
||||
|
||||
// Resolve resolves a path through this node, stopping at any link boundary
|
||||
// and returning the object found as well as the remaining path to traverse
|
||||
func (b *EthHeader) Resolve(p []string) (interface{}, []string, error) {
|
||||
if len(p) == 0 {
|
||||
return b, nil, nil
|
||||
}
|
||||
|
||||
first, rest := p[0], p[1:]
|
||||
|
||||
switch first {
|
||||
case "parent":
|
||||
return &node.Link{Cid: commonHashToCid(MEthHeader, b.ParentHash)}, rest, nil
|
||||
case "receipts":
|
||||
return &node.Link{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, rest, nil
|
||||
case "root":
|
||||
return &node.Link{Cid: commonHashToCid(MEthStateTrie, b.Root)}, rest, nil
|
||||
case "tx":
|
||||
return &node.Link{Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, rest, nil
|
||||
case "uncles":
|
||||
return &node.Link{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, rest, nil
|
||||
}
|
||||
|
||||
if len(p) != 1 {
|
||||
return nil, nil, fmt.Errorf("unexpected path elements past %s", first)
|
||||
}
|
||||
|
||||
switch first {
|
||||
case "bloom":
|
||||
return b.Bloom, nil, nil
|
||||
case "coinbase":
|
||||
return b.Coinbase, nil, nil
|
||||
case "difficulty":
|
||||
return b.Difficulty, nil, nil
|
||||
case "extra":
|
||||
// This is a []byte. By default they are marshalled into Base64.
|
||||
return fmt.Sprintf("0x%x", b.Extra), nil, nil
|
||||
case "gaslimit":
|
||||
return b.GasLimit, nil, nil
|
||||
case "gasused":
|
||||
return b.GasUsed, nil, nil
|
||||
case "mixdigest":
|
||||
return b.MixDigest, nil, nil
|
||||
case "nonce":
|
||||
return b.Nonce, nil, nil
|
||||
case "number":
|
||||
return b.Number, nil, nil
|
||||
case "time":
|
||||
return b.Time, nil, nil
|
||||
default:
|
||||
return nil, nil, ErrInvalidLink
|
||||
}
|
||||
}
|
||||
|
||||
// Tree lists all paths within the object under 'path', and up to the given depth.
|
||||
// To list the entire object (similar to `find .`) pass "" and -1
|
||||
func (b *EthHeader) Tree(p string, depth int) []string {
|
||||
if p != "" || depth == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return []string{
|
||||
"time",
|
||||
"bloom",
|
||||
"coinbase",
|
||||
"difficulty",
|
||||
"extra",
|
||||
"gaslimit",
|
||||
"gasused",
|
||||
"mixdigest",
|
||||
"nonce",
|
||||
"number",
|
||||
"parent",
|
||||
"receipts",
|
||||
"root",
|
||||
"tx",
|
||||
"uncles",
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveLink is a helper function that allows easier traversal of links through blocks
|
||||
func (b *EthHeader) ResolveLink(p []string) (*node.Link, []string, error) {
|
||||
obj, rest, err := b.Resolve(p)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if lnk, ok := obj.(*node.Link); ok {
|
||||
return lnk, rest, nil
|
||||
}
|
||||
|
||||
return nil, nil, fmt.Errorf("resolved item was not a link")
|
||||
}
|
||||
|
||||
// Copy will go away. It is here to comply with the Node interface.
|
||||
func (b *EthHeader) Copy() node.Node {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
// Links is a helper function that returns all links within this object
|
||||
// HINT: Use `ipfs refs <cid>`
|
||||
func (b *EthHeader) Links() []*node.Link {
|
||||
return []*node.Link{
|
||||
{Cid: commonHashToCid(MEthHeader, b.ParentHash)},
|
||||
{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)},
|
||||
{Cid: commonHashToCid(MEthStateTrie, b.Root)},
|
||||
{Cid: commonHashToCid(MEthTxTrie, b.TxHash)},
|
||||
{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)},
|
||||
}
|
||||
}
|
||||
|
||||
// Stat will go away. It is here to comply with the Node interface.
|
||||
func (b *EthHeader) Stat() (*node.NodeStat, error) {
|
||||
return &node.NodeStat{}, nil
|
||||
}
|
||||
|
||||
// Size will go away. It is here to comply with the Node interface.
|
||||
func (b *EthHeader) Size() (uint64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
/*
|
||||
EthHeader functions
|
||||
*/
|
||||
|
||||
// MarshalJSON processes the block header into readable JSON format,
|
||||
// converting the right links into their cids, and keeping the original
|
||||
// hex hash, allowing the user to simplify external queries.
|
||||
func (b *EthHeader) MarshalJSON() ([]byte, error) {
|
||||
out := map[string]interface{}{
|
||||
"time": b.Time,
|
||||
"bloom": b.Bloom,
|
||||
"coinbase": b.Coinbase,
|
||||
"difficulty": b.Difficulty,
|
||||
"extra": fmt.Sprintf("0x%x", b.Extra),
|
||||
"gaslimit": b.GasLimit,
|
||||
"gasused": b.GasUsed,
|
||||
"mixdigest": b.MixDigest,
|
||||
"nonce": b.Nonce,
|
||||
"number": b.Number,
|
||||
"parent": commonHashToCid(MEthHeader, b.ParentHash),
|
||||
"receipts": commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash),
|
||||
"root": commonHashToCid(MEthStateTrie, b.Root),
|
||||
"tx": commonHashToCid(MEthTxTrie, b.TxHash),
|
||||
"uncles": commonHashToCid(MEthHeaderList, b.UncleHash),
|
||||
}
|
||||
return json.Marshal(out)
|
||||
}
|
||||
|
||||
// objJSONHeader defines the output of the JSON RPC API for either
|
||||
// "eth_BlockByHash" or "eth_BlockByHeader".
|
||||
type objJSONHeader struct {
|
||||
Result objJSONHeaderResult `json:"result"`
|
||||
}
|
||||
|
||||
// objJSONBLockResult is the nested struct that takes
|
||||
// the contents of the JSON field "result".
|
||||
type objJSONHeaderResult struct {
|
||||
types.Header // Use its fields and unmarshaler
|
||||
*objJSONHeaderResultExt // Add these fields to the parsing
|
||||
}
|
||||
|
||||
// objJSONBLockResultExt facilitates the composition
|
||||
// of the field "result", adding to the
|
||||
// `types.Header` fields, both ommers (their hashes) and transactions.
|
||||
type objJSONHeaderResultExt struct {
|
||||
OmmerHashes []common.Hash `json:"uncles"`
|
||||
Transactions []*types.Transaction `json:"transactions"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON overrides the function types.Header.UnmarshalJSON, allowing us
|
||||
// to parse the fields of Header, plus ommer hashes and transactions.
|
||||
// (yes, ommer hashes. You will need to "eth_getUncleCountByBlockHash" per each ommer)
|
||||
func (o *objJSONHeaderResult) UnmarshalJSON(input []byte) error {
|
||||
err := o.Header.UnmarshalJSON(input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
o.objJSONHeaderResultExt = &objJSONHeaderResultExt{}
|
||||
err = json.Unmarshal(input, o.objJSONHeaderResultExt)
|
||||
return err
|
||||
}
|
||||
583
statediff/indexer/ipld/eth_header_test.go
Normal file
583
statediff/indexer/ipld/eth_header_test.go
Normal file
@ -0,0 +1,583 @@
|
||||
package ipld
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
block "github.com/ipfs/go-block-format"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
"github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
)
|
||||
|
||||
func TestBlockBodyRlpParsing(t *testing.T) {
|
||||
fi, err := os.Open("test_data/eth-block-body-rlp-999999")
|
||||
checkError(err, t)
|
||||
|
||||
output, _, _, err := FromBlockRLP(fi)
|
||||
checkError(err, t)
|
||||
|
||||
testEthBlockFields(output, t)
|
||||
}
|
||||
|
||||
func TestBlockHeaderRlpParsing(t *testing.T) {
|
||||
fi, err := os.Open("test_data/eth-block-header-rlp-999999")
|
||||
checkError(err, t)
|
||||
|
||||
output, _, _, err := FromBlockRLP(fi)
|
||||
checkError(err, t)
|
||||
|
||||
testEthBlockFields(output, t)
|
||||
}
|
||||
|
||||
func TestBlockBodyJsonParsing(t *testing.T) {
|
||||
fi, err := os.Open("test_data/eth-block-body-json-999999")
|
||||
checkError(err, t)
|
||||
|
||||
output, _, _, err := FromBlockJSON(fi)
|
||||
checkError(err, t)
|
||||
|
||||
testEthBlockFields(output, t)
|
||||
}
|
||||
|
||||
func TestEthBlockProcessTransactionsError(t *testing.T) {
|
||||
// Let's just change one byte in a field of one of these transactions.
|
||||
fi, err := os.Open("test_data/error-tx-eth-block-body-json-999999")
|
||||
checkError(err, t)
|
||||
|
||||
_, _, _, err = FromBlockJSON(fi)
|
||||
if err == nil {
|
||||
t.Fatal("Expected an error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodeBlockHeader should work for both inputs (block header and block body)
|
||||
// as what we are storing is just the block header
|
||||
func TestDecodeBlockHeader(t *testing.T) {
|
||||
storedEthBlock := prepareStoredEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
ethBlock, err := DecodeEthHeader(storedEthBlock.Cid(), storedEthBlock.RawData())
|
||||
checkError(err, t)
|
||||
|
||||
testEthBlockFields(ethBlock, t)
|
||||
}
|
||||
|
||||
func TestEthBlockString(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
if ethBlock.String() != "<EthHeader bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a>" {
|
||||
t.Fatalf("Wrong String()\r\nexpected %s\r\ngot %s", "<EthHeader bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a>", ethBlock.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockLoggable(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
l := ethBlock.Loggable()
|
||||
if _, ok := l["type"]; !ok {
|
||||
t.Fatal("Loggable map expected the field 'type'")
|
||||
}
|
||||
|
||||
if l["type"] != "eth-header" {
|
||||
t.Fatalf("Wrong Loggable 'type' value\r\nexpected %s\r\ngot %s", "eth-header", l["type"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockJSONMarshal(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
jsonOutput, err := ethBlock.MarshalJSON()
|
||||
checkError(err, t)
|
||||
|
||||
var data map[string]interface{}
|
||||
err = json.Unmarshal(jsonOutput, &data)
|
||||
checkError(err, t)
|
||||
|
||||
// Testing all fields is boring, but can help us to avoid
|
||||
// that dreaded regression
|
||||
if data["bloom"].(string)[:10] != "0x00000000" {
|
||||
t.Fatalf("Wrong Bloom\r\nexpected %s\r\ngot %s", "0x00000000", data["bloom"].(string)[:10])
|
||||
t.Fatal("Wrong Bloom")
|
||||
}
|
||||
if data["coinbase"] != "0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5" {
|
||||
t.Fatalf("Wrong coinbase\r\nexpected %s\r\ngot %s", "0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5", data["coinbase"])
|
||||
}
|
||||
if parseFloat(data["difficulty"]) != "12555463106190" {
|
||||
t.Fatalf("Wrong Difficulty\r\nexpected %s\r\ngot %s", "12555463106190", parseFloat(data["difficulty"]))
|
||||
}
|
||||
if data["extra"] != "0xd783010303844765746887676f312e342e32856c696e7578" {
|
||||
t.Fatalf("Wrong Extra\r\nexpected %s\r\ngot %s", "0xd783010303844765746887676f312e342e32856c696e7578", data["extra"])
|
||||
}
|
||||
if parseFloat(data["gaslimit"]) != "3141592" {
|
||||
t.Fatalf("Wrong Gas limit\r\nexpected %s\r\ngot %s", "3141592", parseFloat(data["gaslimit"]))
|
||||
}
|
||||
if parseFloat(data["gasused"]) != "231000" {
|
||||
t.Fatalf("Wrong Gas used\r\nexpected %s\r\ngot %s", "231000", parseFloat(data["gasused"]))
|
||||
}
|
||||
if data["mixdigest"] != "0x5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0" {
|
||||
t.Fatalf("Wrong Mix digest\r\nexpected %s\r\ngot %s", "0x5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0", data["mixdigest"])
|
||||
}
|
||||
if data["nonce"] != "0xf491f46b60fe04b3" {
|
||||
t.Fatalf("Wrong nonce\r\nexpected %s\r\ngot %s", "0xf491f46b60fe04b3", data["nonce"])
|
||||
}
|
||||
if parseFloat(data["number"]) != "999999" {
|
||||
t.Fatalf("Wrong block number\r\nexpected %s\r\ngot %s", "999999", parseFloat(data["number"]))
|
||||
}
|
||||
if parseMapElement(data["parent"]) != "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a" {
|
||||
t.Fatalf("Wrong Parent cid\r\nexpected %s\r\ngot %s", "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a", parseMapElement(data["parent"]))
|
||||
}
|
||||
if parseMapElement(data["receipts"]) != "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq" {
|
||||
t.Fatalf("Wrong Receipt root cid\r\nexpected %s\r\ngot %s", "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq", parseMapElement(data["receipts"]))
|
||||
}
|
||||
if parseMapElement(data["root"]) != "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia" {
|
||||
t.Fatalf("Wrong root hash cid\r\nexpected %s\r\ngot %s", "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia", parseMapElement(data["root"]))
|
||||
}
|
||||
if parseFloat(data["time"]) != "1455404037" {
|
||||
t.Fatalf("Wrong Time\r\nexpected %s\r\ngot %s", "1455404037", parseFloat(data["time"]))
|
||||
}
|
||||
if parseMapElement(data["tx"]) != "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka" {
|
||||
t.Fatalf("Wrong Tx root cid\r\nexpected %s\r\ngot %s", "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka", parseMapElement(data["tx"]))
|
||||
}
|
||||
if parseMapElement(data["uncles"]) != "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq" {
|
||||
t.Fatalf("Wrong Uncle hash cid\r\nexpected %s\r\ngot %s", "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq", parseMapElement(data["uncles"]))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockLinks(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
links := ethBlock.Links()
|
||||
if links[0].Cid.String() != "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a" {
|
||||
t.Fatalf("Wrong cid for parent link\r\nexpected: %s\r\ngot %s", "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a", links[0].Cid.String())
|
||||
}
|
||||
if links[1].Cid.String() != "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq" {
|
||||
t.Fatalf("Wrong cid for receipt root link\r\nexpected: %s\r\ngot %s", "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq", links[1].Cid.String())
|
||||
}
|
||||
if links[2].Cid.String() != "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia" {
|
||||
t.Fatalf("Wrong cid for state root link\r\nexpected: %s\r\ngot %s", "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia", links[2].Cid.String())
|
||||
}
|
||||
if links[3].Cid.String() != "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka" {
|
||||
t.Fatalf("Wrong cid for tx root link\r\nexpected: %s\r\ngot %s", "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka", links[3].Cid.String())
|
||||
}
|
||||
if links[4].Cid.String() != "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq" {
|
||||
t.Fatalf("Wrong cid for uncles root link\r\nexpected: %s\r\ngot %s", "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq", links[4].Cid.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockResolveEmptyPath(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
obj, rest, err := ethBlock.Resolve([]string{})
|
||||
checkError(err, t)
|
||||
|
||||
if ethBlock != obj.(*EthHeader) {
|
||||
t.Fatal("Should have returned the same eth-block object")
|
||||
}
|
||||
|
||||
if len(rest) != 0 {
|
||||
t.Fatalf("Wrong len of rest of the path returned\r\nexpected %d\r\ngot %d", 0, len(rest))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockResolveNoSuchLink(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
_, _, err := ethBlock.Resolve([]string{"wewonthavethisfieldever"})
|
||||
if err == nil {
|
||||
t.Fatal("Should have failed with unknown field")
|
||||
}
|
||||
|
||||
if err != ErrInvalidLink {
|
||||
t.Fatalf("Wrong error message\r\nexpected %s\r\ngot %s", ErrInvalidLink, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockResolveBloom(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
obj, rest, err := ethBlock.Resolve([]string{"bloom"})
|
||||
checkError(err, t)
|
||||
|
||||
// The marshaler of types.Bloom should output it as 0x
|
||||
bloomInText := fmt.Sprintf("%x", obj.(types.Bloom))
|
||||
if bloomInText[:10] != "0000000000" {
|
||||
t.Fatalf("Wrong Bloom\r\nexpected %s\r\ngot %s", "0000000000", bloomInText[:10])
|
||||
}
|
||||
|
||||
if len(rest) != 0 {
|
||||
t.Fatalf("Wrong len of rest of the path returned\r\nexpected %d\r\ngot %d", 0, len(rest))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockResolveBloomExtraPathElements(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
obj, rest, err := ethBlock.Resolve([]string{"bloom", "unexpected", "extra", "elements"})
|
||||
if obj != nil {
|
||||
t.Fatal("Returned obj should be nil")
|
||||
}
|
||||
|
||||
if rest != nil {
|
||||
t.Fatal("Returned rest should be nil")
|
||||
}
|
||||
|
||||
if err.Error() != "unexpected path elements past bloom" {
|
||||
t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "unexpected path elements past bloom", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockResolveNonLinkFields(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
testCases := map[string][]string{
|
||||
"coinbase": {"%x", "52bc44d5378309ee2abf1539bf71de1b7d7be3b5"},
|
||||
"difficulty": {"%s", "12555463106190"},
|
||||
"extra": {"%s", "0xd783010303844765746887676f312e342e32856c696e7578"},
|
||||
"gaslimit": {"%d", "3141592"},
|
||||
"gasused": {"%d", "231000"},
|
||||
"mixdigest": {"%x", "5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0"},
|
||||
"nonce": {"%x", "f491f46b60fe04b3"},
|
||||
"number": {"%s", "999999"},
|
||||
"time": {"%d", "1455404037"},
|
||||
}
|
||||
|
||||
for field, value := range testCases {
|
||||
obj, rest, err := ethBlock.Resolve([]string{field})
|
||||
checkError(err, t)
|
||||
|
||||
format := value[0]
|
||||
result := value[1]
|
||||
if fmt.Sprintf(format, obj) != result {
|
||||
t.Fatalf("Wrong %v\r\nexpected %v\r\ngot %s", field, result, fmt.Sprintf(format, obj))
|
||||
}
|
||||
|
||||
if len(rest) != 0 {
|
||||
t.Fatalf("Wrong len of rest of the path returned\r\nexpected %d\r\ngot %d", 0, len(rest))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockResolveNonLinkFieldsExtraPathElements(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
testCases := []string{
|
||||
"coinbase",
|
||||
"difficulty",
|
||||
"extra",
|
||||
"gaslimit",
|
||||
"gasused",
|
||||
"mixdigest",
|
||||
"nonce",
|
||||
"number",
|
||||
"time",
|
||||
}
|
||||
|
||||
for _, field := range testCases {
|
||||
obj, rest, err := ethBlock.Resolve([]string{field, "unexpected", "extra", "elements"})
|
||||
if obj != nil {
|
||||
t.Fatal("Returned obj should be nil")
|
||||
}
|
||||
|
||||
if rest != nil {
|
||||
t.Fatal("Returned rest should be nil")
|
||||
}
|
||||
|
||||
if err.Error() != "unexpected path elements past "+field {
|
||||
t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "unexpected path elements past "+field, err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockResolveLinkFields(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
testCases := map[string]string{
|
||||
"parent": "bagiacgza2m6j3xu774hlvjxhd2fsnuv5ufom6ei4ply3mm3jrleeozt7b62a",
|
||||
"receipts": "bagkacgzap6qpnsrkagbdecgybaa63ljx4pr2aa5vlsetdg2f5mpzpbrk2iuq",
|
||||
"root": "baglacgza5wmkus23dhec7m2tmtyikcfobjw6yzs7uv3ghxfjjroxavkm3yia",
|
||||
"tx": "bagjacgzair6l3dci6smknejlccbrzx7vtr737s56onoksked2t5anxgxvzka",
|
||||
"uncles": "bagiqcgzadxge32g6y5oxvk4fwvt3ntgudljreri3ssfhie7qufbp2qgusndq",
|
||||
}
|
||||
|
||||
for field, result := range testCases {
|
||||
obj, rest, err := ethBlock.Resolve([]string{field, "anything", "goes", "here"})
|
||||
checkError(err, t)
|
||||
|
||||
lnk, ok := obj.(*node.Link)
|
||||
if !ok {
|
||||
t.Fatal("Returned object is not a link")
|
||||
}
|
||||
|
||||
if lnk.Cid.String() != result {
|
||||
t.Fatalf("Wrong %s cid\r\nexpected %v\r\ngot %v", field, result, lnk.Cid.String())
|
||||
}
|
||||
|
||||
for i, p := range []string{"anything", "goes", "here"} {
|
||||
if rest[i] != p {
|
||||
t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", p, rest[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockTreeBadParams(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
tree := ethBlock.Tree("non-empty-string", 0)
|
||||
if tree != nil {
|
||||
t.Fatal("Expected nil to be returned")
|
||||
}
|
||||
|
||||
tree = ethBlock.Tree("non-empty-string", 1)
|
||||
if tree != nil {
|
||||
t.Fatal("Expected nil to be returned")
|
||||
}
|
||||
|
||||
tree = ethBlock.Tree("", 0)
|
||||
if tree != nil {
|
||||
t.Fatal("Expected nil to be returned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEThBlockTree(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
tree := ethBlock.Tree("", 1)
|
||||
lookupElements := map[string]interface{}{
|
||||
"bloom": nil,
|
||||
"coinbase": nil,
|
||||
"difficulty": nil,
|
||||
"extra": nil,
|
||||
"gaslimit": nil,
|
||||
"gasused": nil,
|
||||
"mixdigest": nil,
|
||||
"nonce": nil,
|
||||
"number": nil,
|
||||
"parent": nil,
|
||||
"receipts": nil,
|
||||
"root": nil,
|
||||
"time": nil,
|
||||
"tx": nil,
|
||||
"uncles": nil,
|
||||
}
|
||||
|
||||
if len(tree) != len(lookupElements) {
|
||||
t.Fatalf("Wrong number of elements\r\nexpected %d\r\ngot %d", len(lookupElements), len(tree))
|
||||
}
|
||||
|
||||
for _, te := range tree {
|
||||
if _, ok := lookupElements[te]; !ok {
|
||||
t.Fatalf("Unexpected Element: %v", te)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The two functions above: TestEthBlockResolveNonLinkFields and
|
||||
// TestEthBlockResolveLinkFields did all the heavy lifting. Then, we will
|
||||
// just test two use cases.
|
||||
func TestEthBlockResolveLinksBadLink(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
obj, rest, err := ethBlock.ResolveLink([]string{"supercalifragilist"})
|
||||
if obj != nil {
|
||||
t.Fatalf("Expected obj to be nil")
|
||||
}
|
||||
if rest != nil {
|
||||
t.Fatal("Expected rest to be nil")
|
||||
}
|
||||
|
||||
if err != ErrInvalidLink {
|
||||
t.Fatalf("Expected error\r\nexpected %s\r\ngot %s", ErrInvalidLink, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockResolveLinksGoodLink(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
obj, rest, err := ethBlock.ResolveLink([]string{"tx", "0", "0", "0"})
|
||||
if obj == nil {
|
||||
t.Fatalf("Expected valid *node.Link obj to be returned")
|
||||
}
|
||||
|
||||
if rest == nil {
|
||||
t.Fatal("Expected rest to be returned")
|
||||
}
|
||||
for i, p := range []string{"0", "0", "0"} {
|
||||
if rest[i] != p {
|
||||
t.Fatalf("Wrong rest of the path returned\r\nexpected %s\r\ngot %s", p, rest[i])
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatal("Non error expected")
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
These functions below should go away
|
||||
We are working on test coverage anyways...
|
||||
*/
|
||||
func TestEthBlockCopy(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
defer func() {
|
||||
r := recover()
|
||||
if r == nil {
|
||||
t.Fatal("Expected panic")
|
||||
}
|
||||
if r != "implement me" {
|
||||
t.Fatalf("Wrong panic message\r\nexpected %s\r\ngot %s", "'implement me'", r)
|
||||
}
|
||||
}()
|
||||
|
||||
_ = ethBlock.Copy()
|
||||
}
|
||||
|
||||
func TestEthBlockStat(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
obj, err := ethBlock.Stat()
|
||||
if obj == nil {
|
||||
t.Fatal("Expected a not null object node.NodeStat")
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatal("Expected a nil error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEthBlockSize(t *testing.T) {
|
||||
ethBlock := prepareDecodedEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
|
||||
size, err := ethBlock.Size()
|
||||
if size != 0 {
|
||||
t.Fatalf("Wrong size\r\nexpected %d\r\ngot %d", 0, size)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatal("Expected a nil error")
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
AUXILIARS
|
||||
*/
|
||||
|
||||
// checkError makes 3 lines into 1.
|
||||
func checkError(err error, t *testing.T) {
|
||||
if err != nil {
|
||||
_, fn, line, _ := runtime.Caller(1)
|
||||
t.Fatalf("[%v:%v] %v", fn, line, err)
|
||||
}
|
||||
}
|
||||
|
||||
// parseFloat is a convenience function to test json output
|
||||
func parseFloat(v interface{}) string {
|
||||
return strconv.FormatFloat(v.(float64), 'f', 0, 64)
|
||||
}
|
||||
|
||||
// parseMapElement is a convenience function to tets json output
|
||||
func parseMapElement(v interface{}) string {
|
||||
return v.(map[string]interface{})["/"].(string)
|
||||
}
|
||||
|
||||
// prepareStoredEthBlock reads the block from a file source to get its rawdata
|
||||
// and computes its cid, for then, feeding it into a new IPLD block function.
|
||||
// So we can pretend that we got this block from the datastore
|
||||
func prepareStoredEthBlock(filepath string, t *testing.T) *block.BasicBlock {
|
||||
// Prepare the "fetched block". This one is supposed to be in the datastore
|
||||
// and given away by github.com/ipfs/go-ipfs/merkledag
|
||||
fi, err := os.Open(filepath)
|
||||
checkError(err, t)
|
||||
|
||||
b, err := ioutil.ReadAll(fi)
|
||||
checkError(err, t)
|
||||
|
||||
c, err := RawdataToCid(MEthHeader, b, multihash.KECCAK_256)
|
||||
checkError(err, t)
|
||||
|
||||
// It's good to clarify that this one below is an IPLD block
|
||||
storedEthBlock, err := block.NewBlockWithCid(b, c)
|
||||
checkError(err, t)
|
||||
|
||||
return storedEthBlock
|
||||
}
|
||||
|
||||
// prepareDecodedEthBlock is more complex than function above, as it stores a
|
||||
// basic block and RLP-decodes it
|
||||
func prepareDecodedEthBlock(filepath string, t *testing.T) *EthHeader {
|
||||
// Get the block from the datastore and decode it.
|
||||
storedEthBlock := prepareStoredEthBlock("test_data/eth-block-header-rlp-999999", t)
|
||||
ethBlock, err := DecodeEthHeader(storedEthBlock.Cid(), storedEthBlock.RawData())
|
||||
checkError(err, t)
|
||||
|
||||
return ethBlock
|
||||
}
|
||||
|
||||
// testEthBlockFields checks the fields of EthBlock one by one.
|
||||
func testEthBlockFields(ethBlock *EthHeader, t *testing.T) {
|
||||
// Was the cid calculated?
|
||||
if ethBlock.Cid().String() != "bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a" {
|
||||
t.Fatalf("Wrong cid\r\nexpected %s\r\ngot %s", "bagiacgzawt5236hkiuvrhfyy4jya3qitlt6icfcqgheew6vsptlraokppm4a", ethBlock.Cid().String())
|
||||
}
|
||||
|
||||
// Do we have the rawdata available?
|
||||
if fmt.Sprintf("%x", ethBlock.RawData()[:10]) != "f90218a0d33c9dde9fff" {
|
||||
t.Fatalf("Wrong Rawdata\r\nexpected %s\r\ngot %s", "f90218a0d33c9dde9fff", fmt.Sprintf("%x", ethBlock.RawData()[:10]))
|
||||
}
|
||||
|
||||
// Proper Fields of types.Header
|
||||
if fmt.Sprintf("%x", ethBlock.ParentHash) != "d33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4" {
|
||||
t.Fatalf("Wrong ParentHash\r\nexpected %s\r\ngot %s", "d33c9dde9fff0ebaa6e71e8b26d2bda15ccf111c7af1b633698ac847667f0fb4", fmt.Sprintf("%x", ethBlock.ParentHash))
|
||||
}
|
||||
if fmt.Sprintf("%x", ethBlock.UncleHash) != "1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" {
|
||||
t.Fatalf("Wrong UncleHash field\r\nexpected %s\r\ngot %s", "1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", fmt.Sprintf("%x", ethBlock.UncleHash))
|
||||
}
|
||||
if fmt.Sprintf("%x", ethBlock.Coinbase) != "52bc44d5378309ee2abf1539bf71de1b7d7be3b5" {
|
||||
t.Fatalf("Wrong Coinbase\r\nexpected %s\r\ngot %s", "52bc44d5378309ee2abf1539bf71de1b7d7be3b5", fmt.Sprintf("%x", ethBlock.Coinbase))
|
||||
}
|
||||
if fmt.Sprintf("%x", ethBlock.Root) != "ed98aa4b5b19c82fb35364f08508ae0a6dec665fa57663dca94c5d70554cde10" {
|
||||
t.Fatalf("Wrong Root\r\nexpected %s\r\ngot %s", "ed98aa4b5b19c82fb35364f08508ae0a6dec665fa57663dca94c5d70554cde10", fmt.Sprintf("%x", ethBlock.Root))
|
||||
}
|
||||
if fmt.Sprintf("%x", ethBlock.TxHash) != "447cbd8c48f498a6912b10831cdff59c7fbfcbbe735ca92883d4fa06dcd7ae54" {
|
||||
t.Fatalf("Wrong TxHash\r\nexpected %s\r\ngot %s", "447cbd8c48f498a6912b10831cdff59c7fbfcbbe735ca92883d4fa06dcd7ae54", fmt.Sprintf("%x", ethBlock.TxHash))
|
||||
}
|
||||
if fmt.Sprintf("%x", ethBlock.ReceiptHash) != "7fa0f6ca2a01823208d80801edad37e3e3a003b55c89319b45eb1f97862ad229" {
|
||||
t.Fatalf("Wrong ReceiptHash\r\nexpected %s\r\ngot %s", "7fa0f6ca2a01823208d80801edad37e3e3a003b55c89319b45eb1f97862ad229", fmt.Sprintf("%x", ethBlock.ReceiptHash))
|
||||
}
|
||||
if len(ethBlock.Bloom) != 256 {
|
||||
t.Fatalf("Wrong Bloom Length\r\nexpected %d\r\ngot %d", 256, len(ethBlock.Bloom))
|
||||
}
|
||||
if fmt.Sprintf("%x", ethBlock.Bloom[71:76]) != "0000000000" { // You wouldn't want me to print out the whole bloom field?
|
||||
t.Fatalf("Wrong Bloom\r\nexpected %s\r\ngot %s", "0000000000", fmt.Sprintf("%x", ethBlock.Bloom[71:76]))
|
||||
}
|
||||
if ethBlock.Difficulty.String() != "12555463106190" {
|
||||
t.Fatalf("Wrong Difficulty\r\nexpected %s\r\ngot %s", "12555463106190", ethBlock.Difficulty.String())
|
||||
}
|
||||
if ethBlock.Number.String() != "999999" {
|
||||
t.Fatalf("Wrong Block Number\r\nexpected %s\r\ngot %s", "999999", ethBlock.Number.String())
|
||||
}
|
||||
if ethBlock.GasLimit != uint64(3141592) {
|
||||
t.Fatalf("Wrong Gas Limit\r\nexpected %d\r\ngot %d", 3141592, ethBlock.GasLimit)
|
||||
}
|
||||
if ethBlock.GasUsed != uint64(231000) {
|
||||
t.Fatalf("Wrong Gas Used\r\nexpected %d\r\ngot %d", 231000, ethBlock.GasUsed)
|
||||
}
|
||||
if ethBlock.Time != uint64(1455404037) {
|
||||
t.Fatalf("Wrong Time\r\nexpected %d\r\ngot %d", 1455404037, ethBlock.Time)
|
||||
}
|
||||
if fmt.Sprintf("%x", ethBlock.Extra) != "d783010303844765746887676f312e342e32856c696e7578" {
|
||||
t.Fatalf("Wrong Extra\r\nexpected %s\r\ngot %s", "d783010303844765746887676f312e342e32856c696e7578", fmt.Sprintf("%x", ethBlock.Extra))
|
||||
}
|
||||
if fmt.Sprintf("%x", ethBlock.Nonce) != "f491f46b60fe04b3" {
|
||||
t.Fatalf("Wrong Nonce\r\nexpected %s\r\ngot %s", "f491f46b60fe04b3", fmt.Sprintf("%x", ethBlock.Nonce))
|
||||
}
|
||||
if fmt.Sprintf("%x", ethBlock.MixDigest) != "5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0" {
|
||||
t.Fatalf("Wrong MixDigest\r\nexpected %s\r\ngot %s", "5b10f4a08a6c209d426f6158bd24b574f4f7b7aa0099c67c14a1f693b4dd04d0", fmt.Sprintf("%x", ethBlock.MixDigest))
|
||||
}
|
||||
}
|
||||
158
statediff/indexer/ipld/eth_log.go
Normal file
158
statediff/indexer/ipld/eth_log.go
Normal file
@ -0,0 +1,158 @@
|
||||
package ipld
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
|
||||
// EthLog (eth-log, codec 0x9a), represents an ethereum block header
|
||||
type EthLog struct {
|
||||
*types.Log
|
||||
|
||||
rawData []byte
|
||||
cid cid.Cid
|
||||
}
|
||||
|
||||
// Static (compile time) check that EthLog satisfies the node.Node interface.
|
||||
var _ node.Node = (*EthLog)(nil)
|
||||
|
||||
// NewLog create a new EthLog IPLD node
|
||||
func NewLog(log *types.Log) (*EthLog, error) {
|
||||
logRaw, err := rlp.EncodeToBytes(log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := RawdataToCid(MEthLog, logRaw, mh.KECCAK_256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &EthLog{
|
||||
Log: log,
|
||||
cid: c,
|
||||
rawData: logRaw,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DecodeEthLogs takes a cid and its raw binary data
|
||||
func DecodeEthLogs(c cid.Cid, b []byte) (*EthLog, error) {
|
||||
l := new(types.Log)
|
||||
if err := rlp.DecodeBytes(b, l); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &EthLog{
|
||||
Log: l,
|
||||
cid: c,
|
||||
rawData: b,
|
||||
}, nil
|
||||
}
|
||||
|
||||
/*
|
||||
Block INTERFACE
|
||||
*/
|
||||
|
||||
// RawData returns the binary of the RLP encode of the log.
|
||||
func (l *EthLog) RawData() []byte {
|
||||
return l.rawData
|
||||
}
|
||||
|
||||
// Cid returns the cid of the receipt log.
|
||||
func (l *EthLog) Cid() cid.Cid {
|
||||
return l.cid
|
||||
}
|
||||
|
||||
// String is a helper for output
|
||||
func (l *EthLog) String() string {
|
||||
return fmt.Sprintf("<EthereumLog %s>", l.cid)
|
||||
}
|
||||
|
||||
// Loggable returns in a map the type of IPLD Link.
|
||||
func (l *EthLog) Loggable() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "eth-log",
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve resolves a path through this node, stopping at any link boundary
|
||||
// and returning the object found as well as the remaining path to traverse
|
||||
func (l *EthLog) Resolve(p []string) (interface{}, []string, error) {
|
||||
if len(p) == 0 {
|
||||
return l, nil, nil
|
||||
}
|
||||
|
||||
if len(p) > 1 {
|
||||
return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0])
|
||||
}
|
||||
|
||||
switch p[0] {
|
||||
case "address":
|
||||
return l.Address, nil, nil
|
||||
case "data":
|
||||
// This is a []byte. By default they are marshalled into Base64.
|
||||
return fmt.Sprintf("0x%x", l.Data), nil, nil
|
||||
case "topics":
|
||||
return l.Topics, nil, nil
|
||||
case "logIndex":
|
||||
return l.Index, nil, nil
|
||||
case "removed":
|
||||
return l.Removed, nil, nil
|
||||
default:
|
||||
return nil, nil, ErrInvalidLink
|
||||
}
|
||||
}
|
||||
|
||||
// Tree lists all paths within the object under 'path', and up to the given depth.
|
||||
// To list the entire object (similar to `find .`) pass "" and -1
|
||||
func (l *EthLog) Tree(p string, depth int) []string {
|
||||
if p != "" || depth == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return []string{
|
||||
"address",
|
||||
"data",
|
||||
"topics",
|
||||
"logIndex",
|
||||
"removed",
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveLink is a helper function that calls resolve and asserts the
|
||||
// output is a link
|
||||
func (l *EthLog) ResolveLink(p []string) (*node.Link, []string, error) {
|
||||
obj, rest, err := l.Resolve(p)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if lnk, ok := obj.(*node.Link); ok {
|
||||
return lnk, rest, nil
|
||||
}
|
||||
|
||||
return nil, nil, fmt.Errorf("resolved item was not a link")
|
||||
}
|
||||
|
||||
// Copy will go away. It is here to comply with the Node interface.
|
||||
func (l *EthLog) Copy() node.Node {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
// Links is a helper function that returns all links within this object
|
||||
func (l *EthLog) Links() []*node.Link {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat will go away. It is here to comply with the interface.
|
||||
func (l *EthLog) Stat() (*node.NodeStat, error) {
|
||||
return &node.NodeStat{}, nil
|
||||
}
|
||||
|
||||
// Size will go away. It is here to comply with the interface.
|
||||
func (l *EthLog) Size() (uint64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
144
statediff/indexer/ipld/eth_log_trie.go
Normal file
144
statediff/indexer/ipld/eth_log_trie.go
Normal file
@ -0,0 +1,144 @@
|
||||
package ipld
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
|
||||
// EthLogTrie (eth-tx-trie codec 0x9p) represents
|
||||
// a node from the transaction trie in ethereum.
|
||||
type EthLogTrie struct {
|
||||
*TrieNode
|
||||
}
|
||||
|
||||
/*
|
||||
OUTPUT
|
||||
*/
|
||||
|
||||
// DecodeEthLogTrie returns an EthLogTrie object from its cid and rawdata.
|
||||
func DecodeEthLogTrie(c cid.Cid, b []byte) (*EthLogTrie, error) {
|
||||
tn, err := decodeTrieNode(c, b, decodeEthLogTrieLeaf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &EthLogTrie{TrieNode: tn}, nil
|
||||
}
|
||||
|
||||
// decodeEthLogTrieLeaf parses a eth-log-trie leaf
|
||||
// from decoded RLP elements
|
||||
func decodeEthLogTrieLeaf(i []interface{}) ([]interface{}, error) {
|
||||
l := new(types.Log)
|
||||
if err := rlp.DecodeBytes(i[1].([]byte), l); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := RawdataToCid(MEthLogTrie, i[1].([]byte), multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []interface{}{
|
||||
i[0].([]byte),
|
||||
&EthLog{
|
||||
Log: l,
|
||||
cid: c,
|
||||
rawData: i[1].([]byte),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
/*
|
||||
Block INTERFACE
|
||||
*/
|
||||
|
||||
// RawData returns the binary of the RLP encode of the transaction.
|
||||
func (t *EthLogTrie) RawData() []byte {
|
||||
return t.rawdata
|
||||
}
|
||||
|
||||
// Cid returns the cid of the transaction.
|
||||
func (t *EthLogTrie) Cid() cid.Cid {
|
||||
return t.cid
|
||||
}
|
||||
|
||||
// String is a helper for output
|
||||
func (t *EthLogTrie) String() string {
|
||||
return fmt.Sprintf("<EthereumLogTrie %s>", t.cid)
|
||||
}
|
||||
|
||||
// Loggable returns in a map the type of IPLD Link.
|
||||
func (t *EthLogTrie) Loggable() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "eth-log-trie",
|
||||
}
|
||||
}
|
||||
|
||||
// logTrie wraps a localTrie for use on the receipt trie.
|
||||
type logTrie struct {
|
||||
*localTrie
|
||||
}
|
||||
|
||||
// newLogTrie initializes and returns a logTrie.
|
||||
func newLogTrie() *logTrie {
|
||||
return &logTrie{
|
||||
localTrie: newLocalTrie(),
|
||||
}
|
||||
}
|
||||
|
||||
// getNodes invokes the localTrie, which computes the root hash of the
|
||||
// log trie and returns its sql keys, to return a slice
|
||||
// of EthLogTrie nodes.
|
||||
func (rt *logTrie) getNodes() ([]node.Node, error) {
|
||||
keys, err := rt.getKeys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]node.Node, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
n, err := rt.getNodeFromDB(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (rt *logTrie) getNodeFromDB(key []byte) (*EthLogTrie, error) {
|
||||
rawdata, err := rt.db.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tn := &TrieNode{
|
||||
cid: keccak256ToCid(MEthLogTrie, key),
|
||||
rawdata: rawdata,
|
||||
}
|
||||
return &EthLogTrie{TrieNode: tn}, nil
|
||||
}
|
||||
|
||||
// getLeafNodes invokes the localTrie, which returns a slice
|
||||
// of EthLogTrie leaf nodes.
|
||||
func (rt *logTrie) getLeafNodes() ([]*EthLogTrie, []*nodeKey, error) {
|
||||
keys, err := rt.getLeafKeys()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out := make([]*EthLogTrie, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
n, err := rt.getNodeFromDB(k.dbKey)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
|
||||
return out, keys, nil
|
||||
}
|
||||
302
statediff/indexer/ipld/eth_parser.go
Normal file
302
statediff/indexer/ipld/eth_parser.go
Normal file
@ -0,0 +1,302 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package ipld
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
"github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
|
||||
// FromBlockRLP takes an RLP message representing
|
||||
// an ethereum block header or body (header, ommers and txs)
|
||||
// to return it as a set of IPLD nodes for further processing.
|
||||
func FromBlockRLP(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
|
||||
// We may want to use this stream several times
|
||||
rawdata, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
// Let's try to decode the received element as a block body
|
||||
var decodedBlock types.Block
|
||||
err = rlp.Decode(bytes.NewBuffer(rawdata), &decodedBlock)
|
||||
if err != nil {
|
||||
if err.Error()[:41] != "rlp: expected input list for types.Header" {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
// Maybe it is just a header... (body sans ommers and txs)
|
||||
var decodedHeader types.Header
|
||||
err := rlp.Decode(bytes.NewBuffer(rawdata), &decodedHeader)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
c, err := RawdataToCid(MEthHeader, rawdata, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
// It was a header
|
||||
return &EthHeader{
|
||||
Header: &decodedHeader,
|
||||
cid: c,
|
||||
rawdata: rawdata,
|
||||
}, nil, nil, nil
|
||||
}
|
||||
|
||||
// This is a block body (header + ommers + txs)
|
||||
// We'll extract the header bits here
|
||||
headerRawData := getRLP(decodedBlock.Header())
|
||||
c, err := RawdataToCid(MEthHeader, headerRawData, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
ethBlock := &EthHeader{
|
||||
Header: decodedBlock.Header(),
|
||||
cid: c,
|
||||
rawdata: headerRawData,
|
||||
}
|
||||
|
||||
// Process the found eth-tx objects
|
||||
ethTxNodes, ethTxTrieNodes, err := processTransactions(decodedBlock.Transactions(),
|
||||
decodedBlock.Header().TxHash[:])
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
return ethBlock, ethTxNodes, ethTxTrieNodes, nil
|
||||
}
|
||||
|
||||
// FromBlockJSON takes the output of an ethereum client JSON API
|
||||
// (i.e. parity or geth) and returns a set of IPLD nodes.
|
||||
func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
|
||||
var obj objJSONHeader
|
||||
dec := json.NewDecoder(r)
|
||||
err := dec.Decode(&obj)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
headerRawData := getRLP(&obj.Result.Header)
|
||||
c, err := RawdataToCid(MEthHeader, headerRawData, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
ethBlock := &EthHeader{
|
||||
Header: &obj.Result.Header,
|
||||
cid: c,
|
||||
rawdata: headerRawData,
|
||||
}
|
||||
|
||||
// Process the found eth-tx objects
|
||||
ethTxNodes, ethTxTrieNodes, err := processTransactions(obj.Result.Transactions,
|
||||
obj.Result.Header.TxHash[:])
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
return ethBlock, ethTxNodes, ethTxTrieNodes, nil
|
||||
}
|
||||
|
||||
// FromBlockAndReceipts takes a block and processes it
|
||||
// to return it a set of IPLD nodes for further processing.
|
||||
func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
|
||||
// Process the header
|
||||
headerNode, err := NewEthHeader(block.Header())
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
// Process the uncles
|
||||
uncleNodes := make([]*EthHeader, len(block.Uncles()))
|
||||
for i, uncle := range block.Uncles() {
|
||||
uncleNode, err := NewEthHeader(uncle)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
uncleNodes[i] = uncleNode
|
||||
}
|
||||
|
||||
// Process the txs
|
||||
txNodes, txTrieNodes, err := processTransactions(block.Transactions(),
|
||||
block.Header().TxHash[:])
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
// Process the receipts and logs
|
||||
rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts,
|
||||
block.Header().ReceiptHash[:])
|
||||
|
||||
return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err
|
||||
}
|
||||
|
||||
// processTransactions will take the found transactions in a parsed block body
|
||||
// to return IPLD node slices for eth-tx and eth-tx-trie
|
||||
func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*EthTx, []*EthTxTrie, error) {
|
||||
var ethTxNodes []*EthTx
|
||||
transactionTrie := newTxTrie()
|
||||
|
||||
for idx, tx := range txs {
|
||||
ethTx, err := NewEthTx(tx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ethTxNodes = append(ethTxNodes, ethTx)
|
||||
if err := transactionTrie.Add(idx, ethTx.RawData()); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if !bytes.Equal(transactionTrie.rootHash(), expectedTxRoot) {
|
||||
return nil, nil, fmt.Errorf("wrong transaction hash computed")
|
||||
}
|
||||
txTrieNodes, err := transactionTrie.getNodes()
|
||||
return ethTxNodes, txTrieNodes, err
|
||||
}
|
||||
|
||||
// processReceiptsAndLogs will take in receipts
|
||||
// to return IPLD node slices for eth-rct, eth-rct-trie, eth-log, eth-log-trie, eth-log-trie-CID, eth-rct-trie-CID
|
||||
func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
|
||||
// Pre allocating memory.
|
||||
ethRctNodes := make([]*EthReceipt, 0, len(rcts))
|
||||
ethLogleafNodeCids := make([][]cid.Cid, 0, len(rcts))
|
||||
ethLogTrieAndLogNodes := make([][]node.Node, 0, len(rcts))
|
||||
|
||||
receiptTrie := NewRctTrie()
|
||||
|
||||
for idx, rct := range rcts {
|
||||
// Process logs for each receipt.
|
||||
logTrieNodes, leafNodeCids, logTrieHash, err := processLogs(rct.Logs)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
rct.LogRoot = logTrieHash
|
||||
ethLogTrieAndLogNodes = append(ethLogTrieAndLogNodes, logTrieNodes)
|
||||
ethLogleafNodeCids = append(ethLogleafNodeCids, leafNodeCids)
|
||||
|
||||
ethRct, err := NewReceipt(rct)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
ethRctNodes = append(ethRctNodes, ethRct)
|
||||
if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if !bytes.Equal(receiptTrie.rootHash(), expectedRctRoot) {
|
||||
return nil, nil, nil, nil, nil, fmt.Errorf("wrong receipt hash computed")
|
||||
}
|
||||
|
||||
rctTrieNodes, err := receiptTrie.GetNodes()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
rctLeafNodes, keys, err := receiptTrie.GetLeafNodes()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
|
||||
for i, rln := range rctLeafNodes {
|
||||
var idx uint
|
||||
|
||||
r := bytes.NewReader(keys[i].TrieKey)
|
||||
err = rlp.Decode(r, &idx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
ethRctleafNodeCids[idx] = rln.Cid()
|
||||
}
|
||||
|
||||
return ethRctNodes, rctTrieNodes, ethLogTrieAndLogNodes, ethLogleafNodeCids, ethRctleafNodeCids, err
|
||||
}
|
||||
|
||||
const keccak256Length = 32
|
||||
|
||||
func processLogs(logs []*types.Log) ([]node.Node, []cid.Cid, common.Hash, error) {
|
||||
logTr := newLogTrie()
|
||||
shortLog := make(map[uint64]*EthLog, len(logs))
|
||||
for idx, log := range logs {
|
||||
logRaw, err := rlp.EncodeToBytes(log)
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
// if len(logRaw) <= keccak256Length it is possible this value's "leaf node"
|
||||
// will be stored in its parent branch but only if len(partialPathOfTheNode) + len(logRaw) <= keccak256Length
|
||||
// But we can't tell what the partial path will be until the trie is Commit()-ed
|
||||
// So wait until we collect all the leaf nodes, and if we are missing any at the indexes we note in shortLogCIDs
|
||||
// we know that these "leaf nodes" were internalized into their parent branch node and we move forward with
|
||||
// using the cid.Cid we cached in shortLogCIDs
|
||||
if len(logRaw) <= keccak256Length {
|
||||
logNode, err := NewLog(log)
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
shortLog[uint64(idx)] = logNode
|
||||
}
|
||||
if err = logTr.Add(idx, logRaw); err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
}
|
||||
|
||||
logTrieNodes, err := logTr.getNodes()
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
|
||||
leafNodes, keys, err := logTr.getLeafNodes()
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
leafNodeCids := make([]cid.Cid, len(logs))
|
||||
for i, ln := range leafNodes {
|
||||
var idx uint
|
||||
|
||||
r := bytes.NewReader(keys[i].TrieKey)
|
||||
err = rlp.Decode(r, &idx)
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
leafNodeCids[idx] = ln.Cid()
|
||||
}
|
||||
// this is where we check which logs <= keccak256Length were actually internalized into parent branch node
|
||||
// and replace those that were with the cid.Cid for the raw log IPLD
|
||||
for i, l := range shortLog {
|
||||
if !leafNodeCids[i].Defined() {
|
||||
leafNodeCids[i] = l.Cid()
|
||||
// if the leaf node was internalized, we append an IPLD for log itself to the list of IPLDs we need to publish
|
||||
logTrieNodes = append(logTrieNodes, l)
|
||||
}
|
||||
}
|
||||
|
||||
return logTrieNodes, leafNodeCids, common.BytesToHash(logTr.rootHash()), err
|
||||
}
|
||||
108
statediff/indexer/ipld/eth_parser_test.go
Normal file
108
statediff/indexer/ipld/eth_parser_test.go
Normal file
@ -0,0 +1,108 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package ipld
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
|
||||
)
|
||||
|
||||
type kind string
|
||||
|
||||
const (
|
||||
legacy kind = "legacy"
|
||||
eip1559 kind = "eip2930"
|
||||
)
|
||||
|
||||
var blockFileNames = []string{
|
||||
"eth-block-12252078",
|
||||
"eth-block-12365585",
|
||||
"eth-block-12365586",
|
||||
}
|
||||
|
||||
var receiptsFileNames = []string{
|
||||
"eth-receipts-12252078",
|
||||
"eth-receipts-12365585",
|
||||
"eth-receipts-12365586",
|
||||
}
|
||||
|
||||
var kinds = []kind{
|
||||
eip1559,
|
||||
eip1559,
|
||||
legacy,
|
||||
}
|
||||
|
||||
type testCase struct {
|
||||
kind kind
|
||||
block *types.Block
|
||||
receipts types.Receipts
|
||||
}
|
||||
|
||||
func loadBlockData(t *testing.T) []testCase {
|
||||
fileDir := "./eip2930_test_data"
|
||||
testCases := make([]testCase, len(blockFileNames))
|
||||
for i, blockFileName := range blockFileNames {
|
||||
blockRLP, err := ioutil.ReadFile(filepath.Join(fileDir, blockFileName))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load blockRLP from file, err %v", err)
|
||||
}
|
||||
block := new(types.Block)
|
||||
if err := rlp.DecodeBytes(blockRLP, block); err != nil {
|
||||
t.Fatalf("failed to decode blockRLP, err %v", err)
|
||||
}
|
||||
receiptsFileName := receiptsFileNames[i]
|
||||
receiptsRLP, err := ioutil.ReadFile(filepath.Join(fileDir, receiptsFileName))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load receiptsRLP from file, err %s", err)
|
||||
}
|
||||
receipts := make(types.Receipts, 0)
|
||||
if err := rlp.DecodeBytes(receiptsRLP, &receipts); err != nil {
|
||||
t.Fatalf("failed to decode receiptsRLP, err %s", err)
|
||||
}
|
||||
testCases[i] = testCase{
|
||||
block: block,
|
||||
receipts: receipts,
|
||||
kind: kinds[i],
|
||||
}
|
||||
}
|
||||
return testCases
|
||||
}
|
||||
|
||||
func TestFromBlockAndReceipts(t *testing.T) {
|
||||
testCases := loadBlockData(t)
|
||||
for _, tc := range testCases {
|
||||
_, _, _, _, _, _, _, _, _, err := FromBlockAndReceipts(tc.block, tc.receipts)
|
||||
if err != nil {
|
||||
t.Fatalf("error generating IPLDs from block and receipts, err %v, kind %s, block hash %s", err, tc.kind, tc.block.Hash())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessLogs(t *testing.T) {
|
||||
logs := []*types.Log{mocks.MockLog1, mocks.MockLog2}
|
||||
nodes, cids, _, err := processLogs(logs)
|
||||
require.NoError(t, err)
|
||||
require.GreaterOrEqual(t, len(nodes), len(logs))
|
||||
require.Equal(t, len(logs), len(cids))
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user