cerc-io/ipld-eth-db-validator
16
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: cerc-io:v5
Branches Tags
cerc-io:v5
cerc-io:v4
cerc-io:sharding
cerc-io:main
cerc-io:v5.0.0-alpha
cerc-io:v4.1.3-alpha
cerc-io:v4.1.2-alpha
cerc-io:v4.1.1-alpha
cerc-io:v4.1.0-alpha
cerc-io:v4.0.2-alpha
cerc-io:v4.0.1-alpha
cerc-io:v3.0.2
cerc-io:v3.0.1
cerc-io:v4.0.0-alpha
cerc-io:v3.0.0
..
compare: cerc-io:sharding
Branches Tags
cerc-io:v5
cerc-io:v4
cerc-io:sharding
cerc-io:main
cerc-io:v5.0.0-alpha
cerc-io:v4.1.3-alpha
cerc-io:v4.1.2-alpha
cerc-io:v4.1.1-alpha
cerc-io:v4.1.0-alpha
cerc-io:v4.0.2-alpha
cerc-io:v4.0.1-alpha
cerc-io:v3.0.2
cerc-io:v3.0.1
cerc-io:v4.0.0-alpha
cerc-io:v3.0.0

No commits in common. "v5" and "sharding" have entirely different histories.
v5 ... sharding

58 changed files with 17907 additions and 5817 deletions
Show Stats Expand all files Collapse all files

6
.github/workflows/on-pr.yml vendored Normal file
View File

@ -0,0 +1,6 @@
name: Run all tests
on: [pull_request]
jobs:
run-tests:
uses: ./.github/workflows/tests.yml

90
.github/workflows/test.yml vendored
View File

@ -1,90 +0,0 @@
name: Unit and integration tests
on:
# workflow_call:
# Job headers are hidden when not top-level - run them directly for readability until fixed:
# https://github.com/go-gitea/gitea/issues/26736
pull_request:
branches: '*'
push:
branches:
- main
- ci-test
env:
SO_VERSION: v1.1.0-36d4969-202407091537
FIXTURENET_ETH_STACKS_REF: main
jobs:
unit-tests:
name: Run unit tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version-file: 'go.mod'
check-latest: true
- name: Run DB container
run: |
docker compose -f test/compose-db.yml up --wait --quiet-pull
- name: Run tests
run: go test -p 1 -v ./pkg/...
integration-tests:
name: Run integration tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version-file: 'go.mod'
check-latest: true
# At present the stock setup-python action fails on Linux/aarch64
# Conditional steps below workaroud this by using deadsnakes for that case only
- name: "Install Python for ARM on Linux"
if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
uses: deadsnakes/action@v3.0.1
with:
python-version: 3.11
- name: "Install Python cases other than ARM on Linux"
if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install stack-orchestrator
uses: actions/checkout@v3
with:
repository: cerc-io/stack-orchestrator
ref: ${{ env.SO_VERSION }}
path: ./stack-orchestrator
- run: pip install ./stack-orchestrator
- name: Clone fixturenet stack repo
uses: actions/checkout@v4
with:
repository: cerc-io/fixturenet-eth-stacks
ref: ${{ env.FIXTURENET_ETH_STACKS_REF }}
path: ./fixturenet-eth-stacks
progress: false
- name: Run testnet stack
env:
CERC_GO_AUTH_TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth
- name: Run contract deployer
run: |
docker compose -f test/compose-deployer.yml up --wait --quiet-pull
- name: Wait for testnet
run: |
# Start validator at current head, but not before Merge (block 1 on test chain)
while
echo "Waiting for chain head to progress..."
height=$(./scripts/get-block-number.sh $ETH_HTTP_PATH)
[[ "$height" -lt 2 ]];
do sleep 5; done
echo "Chain has reached block $height"
echo VALIDATE_FROM_BLOCK=$height >> "$GITHUB_ENV"
- name: Run tests
run: |
go test -v -p 1 ./integration/... -timeout=20m

91
.github/workflows/tests.yml vendored Normal file
View File

@ -0,0 +1,91 @@
name: Test the stack.
on:
workflow_call:
env:
STACK_ORCHESTRATOR_REF: "f2fd766f5400fcb9eb47b50675d2e3b1f2753702"
GO_ETHEREUM_REF: "v1.10.19-statediff-4.1.0-alpha" # Use the tag, we are going to download the bin not build it.
IPLD_ETH_DB_REF: "b59505eab252670c622b42ce60621e9747fb64f9"
jobs:
integrationtest:
name: Run integration tests
env:
GOPATH: /tmp/go
DB_WRITE: true
runs-on: ubuntu-latest
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- uses: actions/setup-go@v3
with:
go-version: ">=1.18.0"
check-latest: true
- uses: actions/checkout@v2
with:
path: "./ipld-eth-db-validator"
- uses: actions/checkout@v2
with:
ref: ${{ env.STACK_ORCHESTRATOR_REF }}
path: "./stack-orchestrator/"
repository: vulcanize/stack-orchestrator
- uses: actions/checkout@v2
with:
ref: ${{ env.IPLD_ETH_DB_REF }}
repository: vulcanize/ipld-eth-db
path: "./ipld-eth-db/"
- name: Create config file
run: |
echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-db-validator/test/contract >> ./config.sh
echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> ./config.sh
echo db_write=$DB_WRITE >> ./config.sh
cat ./config.sh
- name: Download Geth
run: |
cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
wget https://github.com/vulcanize/go-ethereum/releases/download/${{env.GO_ETHEREUM_REF}}/geth-linux-amd64
- name: Run docker compose
run: |
docker-compose \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
--env-file "$GITHUB_WORKSPACE/config.sh" \
up -d --build
- name: Run integration test.
run: |
cd $GITHUB_WORKSPACE/ipld-eth-db-validator
./scripts/run_integration_test.sh
unittest:
name: Run unit tests
runs-on: ubuntu-latest
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- uses: actions/setup-go@v3
with:
go-version: ">=1.18.0"
check-latest: true
- name: Checkout code
uses: actions/checkout@v2
- name: Spin up database
run: |
docker-compose up -d
- name: Run unit tests
run: |
sleep 30
PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test

1
.gitignore vendored
View File

@ -1,3 +1,2 @@
ipld-eth-db-validator ipld-eth-db-validator
.vscode .vscode
.idea

25
Makefile
View File

@ -1,14 +1,21 @@
CONTRACTS_DIR := ./test/contract/contracts BIN = $(GOPATH)/bin
CONTRACTS_OUTPUT_DIR := ./internal/testdata/build BASE = $(GOPATH)/src/$(PACKAGE)
PKGS = go list ./... | grep -v "^vendor/"
CONTRACTS := GLDToken Test # Tools
contracts: $(foreach C,$(CONTRACTS), $(CONTRACTS_OUTPUT_DIR)/$C.bin $(CONTRACTS_OUTPUT_DIR)/$C.abi) .PHONY: integrationtest
.PHONY: contracts integrationtest: | $(GOOSE)
go vet ./...
go fmt ./...
go run github.com/onsi/ginkgo/ginkgo -r test/ -v
test: contracts
go test -p 1 -v ./pkg/...
.PHONY: test .PHONY: test
test: | $(GOOSE)
go vet ./...
go fmt ./...
go run github.com/onsi/ginkgo/ginkgo -r validator_test/ -v
clean: build:
rm $(CONTRACTS_OUTPUT_DIR)/*.bin $(CONTRACTS_OUTPUT_DIR)/*.abi go fmt ./...
GO111MODULE=on go build

151
README.md
View File

@ -1,124 +1,75 @@
# ipld-eth-db-validator - [Validator-README](#validator-readme)
- [Overview](#overview)
- [Intention for the Validator](#intention-for-the-validator)
- [Edge Cases](#edge-cases)
- [Instructions for Testing](#instructions-for-testing)
- [Code Overview](#code-overview)
- [Known Bugs](#known-bugs)
- [Tests on 03/03/22](#tests-on-03-03-22)
- [Set Up](#set-up)
- [Testing Failures](#testing-failures)
> `ipld-eth-db-validator` performs validation checks on indexed Ethereum IPLD objects in a Postgres database: <small><i><a href='http://ecotrust-canada.github.io/markdown-toc/'>Table of contents generated with markdown-toc</a></i></small>
> * Attempt to apply transactions in each block and validate resultant block hash
> * Check referential integrity between IPLD blocks and index tables
## Setup # Overview
Build the binary: This repository contains the validator. The purpose of the validator is to ensure that the data in the Core Postgres database match the data on the blockchain.
```bash # Intention for the Validator
make build
```
## Configuration The perfect scenario for the validator is as follows:
An example config file: 1. The validator will have the capacity to perform historical checks for the Core Postgres database. Users can contain these historical checks to specified configurations (block range).
2. The validator will validate a certain number of trailing blocks, `t`, trailing the head, `n`. Therefore the validator will constantly perform real-time validation starting at `n` and ending at `n - t`.
3. The validator validates the IPLD blocks in the Core Database; it will update the core database to indicate that the validator validated it.
```toml ## Edge Cases
[database]
# db credentials
name = "cerc_public" # DATABASE_NAME
hostname = "localhost" # DATABASE_HOSTNAME
port = 5432 # DATABASE_PORT
user = "vdbm" # DATABASE_USER
password = "..." # DATABASE_PASSWORD
[validate] We must consider the following edge cases for the validator.
# block height to initiate database validation at
fromBlock = 1 # VALIDATE_FROM_BLOCK (default: 1)
# number of blocks to trail behind the head
trail = 64 # VALIDATE_TRAIL (default: 64)
# retry interval after validator has caught up to (head-trail) height (in sec)
retryInterval = 10 # VALIDATE_RETRY_INTERVAL (default: 10)
# whether to perform a statediffing call on a missing block - There are three different data types that the validator must account for.
stateDiffMissingBlock = true # (default: false)
# statediffing call timeout period (in sec)
stateDiffTimeout = 240 # (default: 240)
[ethereum] # Instructions for Testing
# node info
# path to json chain config (optional)
chainConfig = "" # ETH_CHAIN_CONFIG
# eth chain id for config (overridden by chainConfig)
chainID = "1" # ETH_CHAIN_ID (default: 1)
# http RPC endpoint URL for a statediffing node
httpPath = "localhost:8545" # ETH_HTTP_PATH
[prom] Follow steps in [test/README.md](./test/README.md)
# prometheus metrics
metrics = true # PROM_METRICS (default: false)
http = true # PROM_HTTP (default: false)
httpAddr = "0.0.0.0" # PROM_HTTP_ADDR (default: 127.0.0.1)
httpPort = "9001" # PROM_HTTP_PORT (default: 9001)
dbStats = true # PROM_DB_STATS (default: false)
[log] # Code Overview
# log level (trace, debug, info, warn, error, fatal, panic)
level = "info" # LOG_LEVEL (default: info)
# file path for logging, leave unset to log to stdout
file = "" # LOG_FILE_PATH
```
This section will provide some insight into specific files and their purpose.
* The validation process trails behind the latest block number in the database by config parameter `validate.trail`. - `validator_test/chain_maker.go` - This file contains the code for creating a “test” blockchain.
- `validator_test/validator_test.go` - This file contains testing to validate the validator. It leverages `chain_maker.go` to create a blockchain to validate.
- `pkg/validator/validator.go` - This file contains most of the core logic for the validator.
* If the validator has caught up to (head-trail) height, it waits for a configured time interval (`validate.retryInterval`) before again querying the database. # Known Bugs
* If the validator encounters a missing block (gap) in the database, it makes a `writeStateDiffAt` call to the configured statediffing endpoint (`ethereum.httpPath`) if `validate.stateDiffMissingBlock` is set to `true`. Here it is assumed that the statediffing node pointed to is writing out to the database. 1. The validator is improperly handling missing headers from the database.
1. Scenario
1. The IPLD blocks from the mock blockchain are inserted into the Postgres Data.
2. The validator runs, and all tests pass.
3. Users manually remove the last few rows from the database.
4. The validator runs, and all tests pass - This behavior is neither expected nor wanted.
### Local Setup # Tests on 03/03/22
* Create a chain config file `chain.json` according to chain config in genesis json file used by local geth. The tests highlighted below were conducted to validate the initial behavior of the validator.
Example: ## Set Up
```json Below are the steps utilized to set up the test environment.
{
"chainId": 41337,
"homesteadBlock": 0,
"eip150Block": 0,
"eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"eip155Block": 0,
"eip158Block": 0,
"byzantiumBlock": 0,
"constantinopleBlock": 0,
"petersburgBlock": 0,
"istanbulBlock": 0,
"clique": {
"period": 5,
"epoch": 30000
}
}
```
Provide the path to the above file in the config. 1. Run the `scripts/run_integration_test.sh` script.
1. First comment outline 130 to 133 from `validator_test/validator_test.go`
2. Once the code has completed running, comment out lines 55 to 126, 38 to 40, and 42 to 44.
1. Make the following change `db, err = setupDB() --> db, _ = setupDB()`
3. Run the following command: `ginkgo -r validator_test/ -v`
1. All tests should pass
## Usage ## Testing Failures
* Create / update the config file (refer to example config above). Once we had populated the database, we tested for failures.
* Run validator: 1. Removing a Transaction from `transaction_cids` - If we removed a transaction from the database and ran the test, the test would fail. **This is the expected behavior.**
2. Removing Headers from `eth.header_cids`
```bash 1. If we removed a header block sandwiched between two header blocks, the test would fail (For example, we removed the entry for block 4, and the block range is 1-10). **This is the expected behavior.**
./ipld-eth-db-validator stateValidator --config=<config path> 2. If we removed the tail block(s) from the table, the test would pass (For example, we remove the entry for blocks 8, 9, 10, and the block range is 1-10). **This is _not_ the expected behavior.**
```
Example:
```bash
./ipld-eth-db-validator stateValidator --config=environments/example.toml
```
## Monitoring
* Enable metrics using config parameters `prom.metrics` and `prom.http`.
* `ipld-eth-db-validator` exposes following prometheus metrics at `/metrics` endpoint:
* `last_validated_block`: Last validated block number.
* DB stats if `prom.dbStats` set to `true`.
## Tests
* Follow [Test Instructions](./test/README.md) to run unit and integration tests locally.

84
cmd/env.go
View File

@ -1,84 +0,0 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"github.com/spf13/viper"
)
const (
LOG_LEVEL = "LOG_LEVEL"
LOG_FILE_PATH = "LOG_FILE_PATH"
PROM_METRICS = "PROM_METRICS"
PROM_HTTP = "PROM_HTTP"
PROM_HTTP_ADDR = "PROM_HTTP_ADDR"
PROM_HTTP_PORT = "PROM_HTTP_PORT"
PROM_DB_STATS = "PROM_DB_STATS"
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
ETH_CHAIN_CONFIG = "ETH_CHAIN_CONFIG"
ETH_CHAIN_ID = "ETH_CHAIN_ID"
ETH_HTTP_PATH = "ETH_HTTP_PATH"
VALIDATE_FROM_BLOCK = "VALIDATE_FROM_BLOCK"
VALIDATE_TRAIL = "VALIDATE_TRAIL"
VALIDATE_RETRY_INTERVAL = "VALIDATE_RETRY_INTERVAL"
VALIDATE_STATEDIFF_MISSING_BLOCK = "VALIDATE_STATEDIFF_MISSING_BLOCK"
VALIDATE_STATEDIFF_TIMEOUT = "VALIDATE_STATEDIFF_TIMEOUT"
)
// Bind env vars
func init() {
viper.BindEnv("log.level", LOG_LEVEL)
viper.BindEnv("log.file", LOG_FILE_PATH)
viper.BindEnv("prom.metrics", PROM_METRICS)
viper.BindEnv("prom.http", PROM_HTTP)
viper.BindEnv("prom.httpAddr", PROM_HTTP_ADDR)
viper.BindEnv("prom.httpPort", PROM_HTTP_PORT)
viper.BindEnv("prom.dbStats", PROM_DB_STATS)
viper.BindEnv("database.name", DATABASE_NAME)
viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
viper.BindEnv("database.port", DATABASE_PORT)
viper.BindEnv("database.user", DATABASE_USER)
viper.BindEnv("database.password", DATABASE_PASSWORD)
viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
viper.BindEnv("database.maxLifetime", DATABASE_MAX_CONN_LIFETIME)
viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
viper.BindEnv("ethereum.httpPath", ETH_HTTP_PATH)
viper.BindEnv("validate.fromBlock", VALIDATE_FROM_BLOCK)
viper.BindEnv("validate.trail", VALIDATE_TRAIL)
viper.BindEnv("validate.retryInterval", VALIDATE_RETRY_INTERVAL)
viper.BindEnv("validate.stateDiffMissingBlock", VALIDATE_STATEDIFF_MISSING_BLOCK)
viper.BindEnv("validate.stateDiffTimeout", VALIDATE_STATEDIFF_TIMEOUT)
}

78
cmd/root.go
View File

@ -1,31 +1,12 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd package cmd
import ( import (
"fmt"
"os" "os"
"strings" "strings"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/prom"
) )
var ( var (
@ -50,21 +31,23 @@ func Execute() {
} }
func initFunc(cmd *cobra.Command, args []string) { func initFunc(cmd *cobra.Command, args []string) {
ParseLogFlags() logfile := viper.GetString("logfile")
if logfile != "" {
if viper.GetBool("prom.metrics") { file, err := os.OpenFile(logfile,
log.Info("initializing prometheus metrics") os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
prom.Init() if err == nil {
log.Infof("Directing output to %s", logfile)
log.SetOutput(file)
} else {
log.SetOutput(os.Stdout)
log.Info("Failed to log to file, using default stdout")
}
} else {
log.SetOutput(os.Stdout)
} }
if viper.GetBool("prom.http") { if err := logLevel(); err != nil {
addr := fmt.Sprintf( log.Fatal("Could not set log level: ", err)
"%s:%s",
viper.GetString("prom.httpAddr"),
viper.GetString("prom.httpPort"),
)
log.Info("starting prometheus server")
prom.Serve(addr)
} }
} }
@ -74,31 +57,32 @@ func init() {
viper.AutomaticEnv() viper.AutomaticEnv()
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location") rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
rootCmd.PersistentFlags().String("database-name", "cerc_public", "database name") rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
rootCmd.PersistentFlags().Int("database-port", 5432, "database port") rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname") rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
rootCmd.PersistentFlags().String("database-user", "", "database user") rootCmd.PersistentFlags().String("database-user", "", "database user")
rootCmd.PersistentFlags().String("database-password", "", "database password") rootCmd.PersistentFlags().String("database-password", "", "database password")
rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic") rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
rootCmd.PersistentFlags().Bool("prom-metrics", false, "enable prometheus metrics") _ = viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
rootCmd.PersistentFlags().Bool("prom-http", false, "enable prometheus http service")
rootCmd.PersistentFlags().String("prom-httpAddr", "127.0.0.1", "prometheus http host")
rootCmd.PersistentFlags().String("prom-httpPort", "9001", "prometheus http port")
rootCmd.PersistentFlags().Bool("prom-dbStats", false, "enables prometheus db stats")
_ = viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) _ = viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
_ = viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) _ = viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
_ = viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname")) _ = viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
_ = viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user")) _ = viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
_ = viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password")) _ = viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
_ = viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
_ = viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) _ = viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
}
_ = viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("prom-metrics"))
_ = viper.BindPFlag("prom.http", rootCmd.PersistentFlags().Lookup("prom-http")) func logLevel() error {
_ = viper.BindPFlag("prom.httpAddr", rootCmd.PersistentFlags().Lookup("prom-httpAddr")) lvl, err := log.ParseLevel(viper.GetString("log.level"))
_ = viper.BindPFlag("prom.httpPort", rootCmd.PersistentFlags().Lookup("prom-httpPort")) if err != nil {
_ = viper.BindPFlag("prom.dbStats", rootCmd.PersistentFlags().Lookup("prom-dbStats")) return err
}
log.SetLevel(lvl)
if lvl > log.InfoLevel {
log.SetReportCaller(true)
}
log.Info("Log level set to ", lvl.String())
return nil
} }

45
cmd/state_validator.go
View File

@ -1,19 +1,3 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd package cmd
import ( import (
@ -27,7 +11,7 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/validator" "github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
) )
// stateValidatorCmd represents the stateValidator command // stateValidatorCmd represents the stateValidator command
@ -49,10 +33,7 @@ func stateValidator() {
logWithCommand.Fatal(err) logWithCommand.Fatal(err)
} }
service, err := validator.NewService(cfg, nil) service := validator.NewService(cfg, nil)
if err != nil {
logWithCommand.Fatal(err)
}
wg := new(sync.WaitGroup) wg := new(sync.WaitGroup)
wg.Add(1) wg.Add(1)
@ -68,25 +49,17 @@ func stateValidator() {
func init() { func init() {
rootCmd.AddCommand(stateValidatorCmd) rootCmd.AddCommand(stateValidatorCmd)
stateValidatorCmd.PersistentFlags().String("from-block", "1", "block height to initiate state validation") stateValidatorCmd.PersistentFlags().String("block-height", "1", "block height to initiate state validation")
stateValidatorCmd.PersistentFlags().String("trail", "64", "trail of block height to validate") stateValidatorCmd.PersistentFlags().String("trail", "16", "trail of block height to validate")
stateValidatorCmd.PersistentFlags().String("retry-interval", "10s", "retry interval in seconds after validator has caught up to (head-trail) height") stateValidatorCmd.PersistentFlags().String("sleep-interval", "10", "sleep interval in seconds after validator has caught up to (head-trail) height")
stateValidatorCmd.PersistentFlags().Bool("statediff-missing-block", false, "whether to perform a statediffing call on a missing block")
stateValidatorCmd.PersistentFlags().String("statediff-timeout", "240s", "statediffing call timeout period (in sec)")
stateValidatorCmd.PersistentFlags().String("eth-chain-config", "", "path to json chain config") stateValidatorCmd.PersistentFlags().String("chain-config", "", "path to chain config")
stateValidatorCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
stateValidatorCmd.PersistentFlags().String("eth-http-path", "", "http url for a statediffing node")
_ = viper.BindPFlag("validate.fromBlock", stateValidatorCmd.PersistentFlags().Lookup("from-block")) _ = viper.BindPFlag("validate.block-height", stateValidatorCmd.PersistentFlags().Lookup("block-height"))
_ = viper.BindPFlag("validate.trail", stateValidatorCmd.PersistentFlags().Lookup("trail")) _ = viper.BindPFlag("validate.trail", stateValidatorCmd.PersistentFlags().Lookup("trail"))
_ = viper.BindPFlag("validate.retryInterval", stateValidatorCmd.PersistentFlags().Lookup("retry-interval")) _ = viper.BindPFlag("validate.sleepInterval", stateValidatorCmd.PersistentFlags().Lookup("sleep-interval"))
_ = viper.BindPFlag("validate.stateDiffMissingBlock", stateValidatorCmd.PersistentFlags().Lookup("statediff-missing-block"))
_ = viper.BindPFlag("validate.stateDiffTimeout", stateValidatorCmd.PersistentFlags().Lookup("statediff-timeout"))
_ = viper.BindPFlag("ethereum.chainConfig", stateValidatorCmd.PersistentFlags().Lookup("eth-chain-config")) _ = viper.BindPFlag("ethereum.chainConfig", stateValidatorCmd.PersistentFlags().Lookup("chain-config"))
_ = viper.BindPFlag("ethereum.chainID", stateValidatorCmd.PersistentFlags().Lookup("eth-chain-id"))
_ = viper.BindPFlag("ethereum.httpPath", stateValidatorCmd.PersistentFlags().Lookup("eth-http-path"))
} }
func initConfig() { func initConfig() {

35
cmd/util.go
View File

@ -1,35 +0,0 @@
package cmd
import (
"os"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
func ParseLogFlags() {
logfile := viper.GetString("log.file")
if logfile != "" {
file, err := os.OpenFile(logfile,
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err == nil {
log.Infof("Directing output to %s", logfile)
log.SetOutput(file)
} else {
log.SetOutput(os.Stdout)
log.Info("Failed to log to file, using default stdout")
}
} else {
log.SetOutput(os.Stdout)
}
lvl, err := log.ParseLevel(viper.GetString("log.level"))
if err != nil {
log.Fatal("Could not parse log level: ", err)
}
log.SetLevel(lvl)
if lvl > log.InfoLevel {
log.SetReportCaller(true)
}
log.Info("Log level set to ", lvl)
}

14
test/compose-db.yml → docker-compose.yml
View File

@ -1,26 +1,30 @@
# Containers to run backing DB for unit testing version: '3.2'
services: services:
migrations: migrations:
restart: on-failure restart: on-failure
depends_on: depends_on:
- ipld-eth-db - ipld-eth-db
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.3.0-alpha image: vulcanize/ipld-eth-db:v4.2.0-alpha
environment: environment:
DATABASE_USER: "vdbm" DATABASE_USER: "vdbm"
DATABASE_NAME: "cerc_testing" DATABASE_NAME: "vulcanize_testing"
DATABASE_PASSWORD: "password" DATABASE_PASSWORD: "password"
DATABASE_HOSTNAME: "ipld-eth-db" DATABASE_HOSTNAME: "ipld-eth-db"
DATABASE_PORT: 5432 DATABASE_PORT: 5432
ipld-eth-db: ipld-eth-db:
container_name: test-ipld-eth-db
image: timescale/timescaledb:latest-pg14 image: timescale/timescaledb:latest-pg14
restart: always restart: always
command: ["postgres", "-c", "log_statement=all"] command: ["postgres", "-c", "log_statement=all"]
environment: environment:
POSTGRES_USER: "vdbm" POSTGRES_USER: "vdbm"
POSTGRES_DB: "cerc_testing" POSTGRES_DB: "vulcanize_testing"
POSTGRES_PASSWORD: "password" POSTGRES_PASSWORD: "password"
ports: ports:
- "127.0.0.1:8077:5432" - "127.0.0.1:8077:5432"
volumes:
- vdb_db_eth_validator:/var/lib/postgresql/data
volumes:
vdb_db_eth_validator:

26
environments/example.toml
View File

@ -1,30 +1,14 @@
[database] [database]
name = "cerc_public" name = "vulcanize_public"
hostname = "localhost" hostname = "localhost"
port = 5432 port = 5432
password = "password" password = "password"
user = "vdbm" user = "vdbm"
[validate] [validate]
fromBlock = 1 block-height = 1
trail = 64 trail = 16
retryInterval = "10s" sleepInterval = 10
stateDiffMissingBlock = true
stateDiffTimeout = "240s"
[ethereum] [ethereum]
chainConfig = "" chainConfig = "./chain.json"
chainID = "1"
httpPath = "localhost:8545"
wsPath = "localhost:8546"
[prom]
metrics = true
http = true
httpAddr = "localhost"
httpPort = "9001"
dbStats = true
[log]
file = ""
level = "info"

440
go.mod
View File

@ -1,274 +1,278 @@
module github.com/cerc-io/ipld-eth-db-validator/v5 module github.com/vulcanize/ipld-eth-db-validator
go 1.21 go 1.18
require ( require (
github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha github.com/ethereum/go-ethereum v1.10.19
github.com/cerc-io/ipld-eth-server/v5 v5.3.0-alpha
github.com/cerc-io/ipld-eth-statedb v0.1.1
github.com/cerc-io/plugeth-statediff v0.3.2
github.com/ethereum/go-ethereum v1.13.14
github.com/holiman/uint256 v1.2.4
github.com/jmoiron/sqlx v1.3.5 github.com/jmoiron/sqlx v1.3.5
github.com/onsi/ginkgo/v2 v2.15.0 github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.30.0 github.com/onsi/gomega v1.19.0
github.com/prometheus/client_golang v1.18.0 github.com/sirupsen/logrus v1.8.1
github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.4.0
github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.11.0
github.com/spf13/viper v1.18.2 github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha
github.com/vulcanize/ipld-eth-server/v4 v4.1.0-alpha
) )
require ( require (
bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect
github.com/DataDog/zstd v1.5.5 // indirect github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
github.com/Jorropo/jsync v1.0.1 // indirect github.com/Stebalien/go-bitfield v0.0.1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
github.com/VictoriaMetrics/fastcache v1.12.2 // indirect github.com/benbjohnson/clock v1.1.0 // indirect
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.10.0 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect github.com/btcsuite/btcd v0.22.1 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/cerc-io/eth-iterator-utils v0.3.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cheekybits/genny v1.0.0 // indirect
github.com/cockroachdb/errors v1.10.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/consensys/bavard v0.1.13 // indirect
github.com/consensys/gnark-crypto v0.12.1 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
github.com/cskr/pubsub v1.0.2 // indirect github.com/cskr/pubsub v1.0.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/deckarep/golang-set/v2 v2.3.0 // indirect github.com/deckarep/golang-set v1.8.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/docker/go-units v0.5.0 // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
github.com/flynn/noise v1.1.0 // indirect github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect
github.com/georgysavva/scany v0.2.9 // indirect github.com/georgysavva/scany v0.2.9 // indirect
github.com/getsentry/sentry-go v0.22.0 // indirect github.com/go-ole/go-ole v1.2.1 // indirect
github.com/go-logr/logr v1.4.1 // indirect github.com/go-stack/stack v1.8.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-stack/stack v1.8.1 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.6.0 // indirect github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect github.com/google/uuid v1.3.0 // indirect
github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect
github.com/gorilla/websocket v1.5.1 // indirect github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-bexpr v0.1.10 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
github.com/huin/goupnp v1.3.0 // indirect github.com/holiman/uint256 v1.2.0 // indirect
github.com/inconshreveable/log15 v2.16.0+incompatible // indirect github.com/huin/goupnp v1.0.3 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/boxo v0.19.0 // indirect github.com/ipfs/go-bitswap v0.4.0 // indirect
github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect
github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-blockservice v0.1.7 // indirect
github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-cid v0.0.7 // indirect
github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-cidutil v0.0.2 // indirect
github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-datastore v0.4.6 // indirect
github.com/ipfs/go-ds-measure v0.2.0 // indirect github.com/ipfs/go-ds-measure v0.1.0 // indirect
github.com/ipfs/go-fetcher v1.5.0 // indirect
github.com/ipfs/go-filestore v1.0.0 // indirect
github.com/ipfs/go-fs-lock v0.0.7 // indirect github.com/ipfs/go-fs-lock v0.0.7 // indirect
github.com/ipfs/go-graphsync v0.8.0 // indirect
github.com/ipfs/go-ipfs v0.10.0 // indirect
github.com/ipfs/go-ipfs-blockstore v1.0.1 // indirect
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
github.com/ipfs/go-ipfs-config v0.16.0 // indirect
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-ds-help v1.0.0 // indirect
github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.0.1 // indirect
github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect github.com/ipfs/go-ipfs-exchange-offline v0.0.1 // indirect
github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipfs-files v0.0.8 // indirect
github.com/ipfs/go-ipld-cbor v0.1.0 // indirect github.com/ipfs/go-ipfs-keystore v0.0.2 // indirect
github.com/ipfs/go-ipld-format v0.6.0 // indirect github.com/ipfs/go-ipfs-pinner v0.1.2 // indirect
github.com/ipfs/go-ipld-legacy v0.2.1 // indirect github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
github.com/ipfs/go-ipfs-provider v0.6.1 // indirect
github.com/ipfs/go-ipfs-routing v0.1.0 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
github.com/ipfs/go-ipld-format v0.2.0 // indirect
github.com/ipfs/go-ipld-legacy v0.1.0 // indirect
github.com/ipfs/go-ipns v0.1.2 // indirect
github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/ipfs/go-log/v2 v2.3.0 // indirect
github.com/ipfs/go-merkledag v0.4.0 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-mfs v0.1.2 // indirect
github.com/ipfs/go-unixfsnode v1.9.0 // indirect github.com/ipfs/go-namesys v0.3.1 // indirect
github.com/ipfs/kubo v0.27.0 // indirect github.com/ipfs/go-path v0.1.2 // indirect
github.com/ipld/go-car/v2 v2.13.1 // indirect github.com/ipfs/go-peertaskqueue v0.4.0 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipfs/go-unixfs v0.2.5 // indirect
github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/ipfs/go-unixfsnode v1.1.3 // indirect
github.com/ipfs/go-verifcid v0.0.1 // indirect
github.com/ipfs/interface-go-ipfs-core v0.5.1 // indirect
github.com/ipld/go-codec-dagpb v1.3.0 // indirect
github.com/ipld/go-ipld-prime v0.12.2 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.14.3 // indirect github.com/jackc/pgconn v1.12.1 // indirect
github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.3 // indirect github.com/jackc/pgproto3/v2 v2.3.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/pgtype v1.11.0 // indirect
github.com/jackc/pgx/v4 v4.18.3 // indirect github.com/jackc/pgx/v4 v4.16.1 // indirect
github.com/jackc/puddle v1.3.0 // indirect github.com/jackc/puddle v1.2.1 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect github.com/jbenet/goprocess v0.1.4 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.4 // indirect github.com/jinzhu/now v1.1.4 // indirect
github.com/klauspost/compress v1.17.6 // indirect github.com/klauspost/compress v1.11.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect github.com/koron/go-ssdp v0.0.2 // indirect
github.com/kr/pretty v0.3.1 // indirect github.com/lib/pq v1.10.6 // indirect
github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-addr-util v0.1.0 // indirect
github.com/lib/pq v1.10.9 // indirect github.com/libp2p/go-buffer-pool v0.0.2 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-doh-resolver v0.4.0 // indirect github.com/libp2p/go-conn-security-multistream v0.2.1 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-doh-resolver v0.3.1 // indirect
github.com/libp2p/go-libp2p v0.33.0 // indirect github.com/libp2p/go-eventbus v0.2.1 // indirect
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-flow-metrics v0.0.3 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.24.4 // indirect github.com/libp2p/go-libp2p v0.15.0 // indirect
github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 // indirect
github.com/libp2p/go-libp2p-pubsub v0.10.0 // indirect github.com/libp2p/go-libp2p-autonat v0.4.2 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect github.com/libp2p/go-libp2p-blankhost v0.2.0 // indirect
github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-libp2p-circuit v0.4.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.3 // indirect github.com/libp2p/go-libp2p-connmgr v0.2.4 // indirect
github.com/libp2p/go-libp2p-xor v0.1.0 // indirect github.com/libp2p/go-libp2p-core v0.9.0 // indirect
github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-libp2p-discovery v0.5.1 // indirect
github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.13.1 // indirect
github.com/libp2p/go-netroute v0.2.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect
github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-libp2p-loggables v0.1.0 // indirect
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/libp2p/go-libp2p-mplex v0.4.1 // indirect
github.com/libp2p/zeroconf/v2 v2.2.0 // indirect github.com/libp2p/go-libp2p-nat v0.0.6 // indirect
github.com/magiconair/properties v1.8.7 // indirect github.com/libp2p/go-libp2p-noise v0.2.2 // indirect
github.com/libp2p/go-libp2p-peerstore v0.2.8 // indirect
github.com/libp2p/go-libp2p-pnet v0.2.0 // indirect
github.com/libp2p/go-libp2p-pubsub v0.5.4 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.4.0 // indirect
github.com/libp2p/go-libp2p-quic-transport v0.12.0 // indirect
github.com/libp2p/go-libp2p-record v0.1.3 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.2.3 // indirect
github.com/libp2p/go-libp2p-swarm v0.5.3 // indirect
github.com/libp2p/go-libp2p-tls v0.2.0 // indirect
github.com/libp2p/go-libp2p-transport-upgrader v0.4.6 // indirect
github.com/libp2p/go-libp2p-xor v0.0.0-20210714161855-5c005aca55db // indirect
github.com/libp2p/go-libp2p-yamux v0.5.4 // indirect
github.com/libp2p/go-maddr-filter v0.1.0 // indirect
github.com/libp2p/go-mplex v0.3.0 // indirect
github.com/libp2p/go-msgio v0.0.6 // indirect
github.com/libp2p/go-nat v0.0.5 // indirect
github.com/libp2p/go-netroute v0.1.6 // indirect
github.com/libp2p/go-openssl v0.0.7 // indirect
github.com/libp2p/go-reuseport v0.0.2 // indirect
github.com/libp2p/go-reuseport-transport v0.0.5 // indirect
github.com/libp2p/go-sockaddr v0.1.1 // indirect
github.com/libp2p/go-stream-muxer-multistream v0.3.0 // indirect
github.com/libp2p/go-tcp-transport v0.2.8 // indirect
github.com/libp2p/go-ws-transport v0.5.0 // indirect
github.com/libp2p/go-yamux/v2 v2.2.0 // indirect
github.com/libp2p/zeroconf/v2 v2.0.0 // indirect
github.com/lucas-clemente/quic-go v0.26.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailgun/groupcache/v2 v2.3.0 // indirect github.com/mailgun/groupcache/v2 v2.3.0 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/miekg/dns v1.1.58 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/miekg/dns v1.1.43 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/minio/sha256-simd v1.0.1 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base32 v0.0.3 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-base36 v0.1.0 // indirect
github.com/multiformats/go-multiaddr v0.12.2 // indirect github.com/multiformats/go-multiaddr v0.4.0 // indirect
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multibase v0.0.3 // indirect
github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multicodec v0.3.0 // indirect
github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multihash v0.1.0 // indirect
github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-multistream v0.2.2 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect github.com/multiformats/go-varint v0.0.6 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opencontainers/runtime-spec v1.2.0 // indirect
github.com/openrelayxyz/plugeth-utils v1.5.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
github.com/pganalyze/pg_query_go/v4 v4.2.1 // indirect
github.com/pion/datachannel v1.5.5 // indirect
github.com/pion/dtls/v2 v2.2.8 // indirect
github.com/pion/ice/v2 v2.3.11 // indirect
github.com/pion/interceptor v0.1.25 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns v0.0.9 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.13 // indirect
github.com/pion/rtp v1.8.3 // indirect
github.com/pion/sctp v1.8.9 // indirect
github.com/pion/sdp/v3 v3.0.6 // indirect
github.com/pion/srtp/v2 v2.0.18 // indirect
github.com/pion/stun v0.6.1 // indirect
github.com/pion/transport/v2 v2.2.4 // indirect
github.com/pion/turn/v2 v2.1.4 // indirect
github.com/pion/webrtc/v3 v3.2.23 // indirect
github.com/pkg/errors v0.9.1 // indirect github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/common v0.47.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/common v0.30.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect
github.com/quic-go/quic-go v0.41.0 // indirect github.com/prometheus/tsdb v0.7.1 // indirect
github.com/quic-go/webtransport-go v0.6.0 // indirect github.com/rjeczalik/notify v0.9.1 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rs/cors v1.7.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/samber/lo v1.39.0 // indirect
github.com/segmentio/fasthash v1.0.3 // indirect github.com/segmentio/fasthash v1.0.3 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shirou/gopsutil v3.21.5+incompatible // indirect
github.com/shopspring/decimal v1.2.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.11.0 // indirect github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/status-im/keycard-go v0.2.0 // indirect github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
github.com/stretchr/objx v0.5.2 // indirect github.com/stretchr/objx v0.2.0 // indirect
github.com/stretchr/testify v1.9.0 // indirect github.com/stretchr/testify v1.7.1 // indirect
github.com/subosito/gotenv v1.6.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect
github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/thoas/go-funk v0.9.2 // indirect
github.com/thoas/go-funk v0.9.3 // indirect github.com/tklauser/go-sysconf v0.3.6 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.2.2 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.3-alpha // indirect
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20240109153615-66e95c3e8a87 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 // indirect
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
go.opencensus.io v0.24.0 // indirect go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/otel v1.25.0 // indirect go.opentelemetry.io/otel v0.20.0 // indirect
go.opentelemetry.io/otel/metric v1.25.0 // indirect go.opentelemetry.io/otel/metric v0.20.0 // indirect
go.opentelemetry.io/otel/trace v1.25.0 // indirect go.opentelemetry.io/otel/trace v0.20.0 // indirect
go.uber.org/dig v1.17.1 // indirect go.uber.org/atomic v1.9.0 // indirect
go.uber.org/fx v1.20.1 // indirect go.uber.org/dig v1.10.0 // indirect
go.uber.org/mock v0.4.0 // indirect go.uber.org/fx v1.13.1 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.27.0 // indirect go.uber.org/zap v1.19.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/crypto v0.22.0 // indirect golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
golang.org/x/net v0.24.0 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.3.7 // indirect
golang.org/x/term v0.19.0 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.1.8 // indirect
golang.org/x/tools v0.20.0 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/protobuf v1.28.0 // indirect
gonum.org/v1/gonum v0.14.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect
google.golang.org/protobuf v1.32.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/urfave/cli.v1 v1.20.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gorm.io/driver/postgres v1.3.7 // indirect gorm.io/driver/postgres v1.3.7 // indirect
gorm.io/gorm v1.23.5 // indirect gorm.io/gorm v1.23.5 // indirect
lukechampine.com/blake3 v1.2.2 // indirect lukechampine.com/blake3 v1.1.6 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
) )
replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha

1983
go.sum
View File

File diff suppressed because it is too large Load Diff

64
integration/deploy_helpers.go
View File

@ -1,64 +0,0 @@
package integration
import (
"encoding/json"
"fmt"
"math/big"
"net/http"
)
type ContractDeployed struct {
Address string `json:"address"`
TransactionHash string `json:"txHash"`
BlockNumber uint64 `json:"blockNumber"`
BlockHash string `json:"blockHash"`
}
type ContractDestroyed struct {
BlockNumber uint64 `json:"blockNumber"`
}
type PutResult struct {
BlockNumber uint64 `json:"blockNumber"`
}
type Tx struct {
From string `json:"from"`
To string `json:"to"`
Value *big.Int `json:"value"`
TransactionHash string `json:"txHash"`
BlockNumber uint64 `json:"blockNumber"`
BlockHash string `json:"blockHash"`
}
const ContractServerUrl = "http://localhost:3000"
// Factory to generate endpoint functions
func MakeGetAndDecodeFunc[R any](format string) func(...interface{}) (*R, error) {
return func(params ...interface{}) (*R, error) {
params = append([]interface{}{ContractServerUrl}, params...)
url := fmt.Sprintf(format, params...)
res, err := http.Get(url)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s: %s", url, res.Status)
}
var data R
decoder := json.NewDecoder(res.Body)
return &data, decoder.Decode(&data)
}
}
var (
SendEth = MakeGetAndDecodeFunc[Tx]("%s/v1/sendEth?to=%s&value=%s")
DeployContract = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/deployContract")
DestroyContract = MakeGetAndDecodeFunc[ContractDestroyed]("%s/v1/destroyContract?addr=%s")
DeployTestContract = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/deployTestContract")
DestroyTestContract = MakeGetAndDecodeFunc[ContractDestroyed]("%s/v1/destroyTestContract?addr=%s")
PutTestValue = MakeGetAndDecodeFunc[PutResult]("%s/v1/putTestValue?addr=%s&value=%d")
)

140
integration/integration_test.go
View File

@ -1,140 +0,0 @@
package integration_test
import (
"context"
"sync"
"testing"
"time"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/onsi/gomega"
. "github.com/onsi/gomega"
"github.com/cerc-io/ipld-eth-db-validator/v5/cmd" // this registers env vars with viper
"github.com/cerc-io/ipld-eth-db-validator/v5/integration"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/validator"
)
const (
timeout = 20 * time.Minute
pollInterval = time.Second
progressBufferSize = 200
)
var (
testAddresses = []string{
"0x1111111111111111111111111111111111111112",
"0x1ca7c995f8eF0A2989BbcE08D5B7Efe50A584aa1",
"0x9a4b666af23a2cdb4e5538e1d222a445aeb82134",
"0xF7C7AEaECD2349b129d5d15790241c32eeE4607B",
"0x992b6E9BFCA1F7b0797Cee10b0170E536EAd3532",
}
// Track the blocks validated on this chain
lastValidated uint64
validated = newBlockSet()
ctx = context.Background()
wg sync.WaitGroup
)
func init() {
cmd.ParseLogFlags()
gomega.SetDefaultEventuallyTimeout(timeout)
gomega.SetDefaultEventuallyPollingInterval(pollInterval)
}
func setup(t *testing.T, progressChan chan uint64) {
cfg, err := validator.NewConfig()
if err != nil {
t.Fatal(err)
}
// set the default DB config to the testing defaults
cfg.DBConfig, _ = postgres.TestConfig.WithEnv()
// update the start block if we have already validated past it
if lastValidated > cfg.FromBlock {
cfg.FromBlock = lastValidated
}
// default trail is unnecessarily long
cfg.Trail = 8
service, err := validator.NewService(cfg, progressChan)
if err != nil {
t.Fatal(err)
}
go func() {
for block := range progressChan {
validated.add(block)
lastValidated = block
}
}()
wg.Add(1)
go service.Start(ctx, &wg)
t.Cleanup(func() {
service.Stop()
wg.Wait()
g := gomega.NewWithT(t)
g.Expect(progressChan).To(BeClosed())
})
}
func TestValidateContracts(t *testing.T) {
progressChan := make(chan uint64, progressBufferSize)
setup(t, progressChan)
contract, err := integration.DeployTestContract()
if err != nil {
t.Fatal(err)
}
t.Run("contract deployment", func(t *testing.T) {
g := gomega.NewWithT(t)
t.Logf("Deployed contract at block %d", contract.BlockNumber)
g.Expect(progressChan).ToNot(BeClosed())
g.Eventually(validated.contains, timeout).WithArguments(contract.BlockNumber).Should(BeTrue())
})
t.Run("contract method calls", func(t *testing.T) {
g := gomega.NewWithT(t)
var blocks []uint64
for i := 0; i < 3; i++ {
res, err := integration.PutTestValue(contract.Address, i)
if err != nil {
t.Fatal(err)
}
t.Logf("Put() called at block %d", res.BlockNumber)
blocks = append(blocks, res.BlockNumber)
}
g.Expect(progressChan).ToNot(BeClosed())
g.Eventually(validated.containsAll, timeout).WithArguments(blocks).Should(BeTrue())
})
}
func TestValidateTransactions(t *testing.T) {
progressChan := make(chan uint64, progressBufferSize)
setup(t, progressChan)
t.Run("ETH transfer transactions", func(t *testing.T) {
g := gomega.NewWithT(t)
var blocks []uint64
for _, address := range testAddresses {
tx, err := integration.SendEth(address, "0.01")
if err != nil {
t.Fatal(err)
}
t.Logf("Sent tx at block %d", tx.BlockNumber)
blocks = append(blocks, tx.BlockNumber)
}
g.Expect(progressChan).ToNot(BeClosed())
g.Eventually(validated.containsAll, timeout).WithArguments(blocks).Should(BeTrue())
})
}

36
integration/util_test.go
View File

@ -1,36 +0,0 @@
package integration_test
import (
"sync"
)
type atomicBlockSet struct {
blocks map[uint64]struct{}
sync.RWMutex
}
func newBlockSet() *atomicBlockSet {
return &atomicBlockSet{blocks: make(map[uint64]struct{})}
}
func (set *atomicBlockSet) contains(block uint64) bool {
set.RLock()
defer set.RUnlock()
_, has := set.blocks[block]
return has
}
func (set *atomicBlockSet) containsAll(blocks []uint64) bool {
for _, block := range blocks {
if !set.contains(block) {
return false
}
}
return true
}
func (set *atomicBlockSet) add(block uint64) {
set.Lock()
defer set.Unlock()
set.blocks[block] = struct{}{}
}

100
internal/chaingen/makers.go
View File

@ -1,100 +0,0 @@
package chaingen
import (
"math/big"
"github.com/cerc-io/plugeth-statediff/test_helpers"
"github.com/cerc-io/plugeth-statediff/test_helpers/chaingen"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/cerc-io/ipld-eth-db-validator/v5/internal/testdata"
)
var (
bank, acct1, acct2 common.Address
contractAddr common.Address
contractDataRoot string
defaultContract *chaingen.ContractSpec
)
func init() {
var err error
defaultContract, err = chaingen.ParseContract(testdata.TestContractABI, testdata.TestContractCode)
if err != nil {
panic(err)
}
}
// A GenContext which exactly replicates the chain generator used in existing tests
func DefaultGenContext(chainConfig *params.ChainConfig, db ethdb.Database) *chaingen.GenContext {
gen := chaingen.NewGenContext(chainConfig, db)
bank = gen.AddOwnedAccount(test_helpers.TestBankKey)
acct1 = gen.AddOwnedAccount(test_helpers.Account1Key)
acct2 = gen.AddOwnedAccount(test_helpers.Account2Key)
gen.AddContract("Test", defaultContract)
gen.AddFunction(func(i int, block *core.BlockGen) {
if err := defaultChainGen(gen, i, block); err != nil {
panic(err)
}
})
gen.Genesis = test_helpers.GenesisBlockForTesting(
db, bank, test_helpers.TestBankFunds, big.NewInt(params.InitialBaseFee), params.MaxGasLimit,
)
return gen
}
func defaultChainGen(gen *chaingen.GenContext, i int, block *core.BlockGen) error {
switch i {
case 0:
// In block 1, the test bank sends account #1 some ether.
tx, err := gen.CreateSendTx(bank, acct1, big.NewInt(10000))
if err != nil {
panic(err)
}
block.AddTx(tx)
case 1:
// In block 2, the test bank sends some more ether to account #1.
// acct1 passes it on to account #2.
// acct1 creates a test contract.
tx1, err := gen.CreateSendTx(bank, acct1, big.NewInt(1000))
if err != nil {
panic(err)
}
block.AddTx(tx1)
tx2, err := gen.CreateSendTx(acct1, acct2, big.NewInt(1000))
if err != nil {
panic(err)
}
block.AddTx(tx2)
contractAddr, err = gen.DeployContract(acct1, "Test")
if err != nil {
panic(err)
}
case 2:
block.SetCoinbase(acct2)
tx, err := gen.CreateCallTx(bank, contractAddr, "Put", big.NewInt(3))
if err != nil {
panic(err)
}
block.AddTx(tx)
case 3:
block.SetCoinbase(acct2)
tx, err := gen.CreateCallTx(bank, contractAddr, "Put", big.NewInt(9))
if err != nil {
panic(err)
}
block.AddTx(tx)
case 4:
block.SetCoinbase(acct1)
tx, err := gen.CreateCallTx(bank, contractAddr, "Put", big.NewInt(0))
if err != nil {
panic(err)
}
block.AddTx(tx)
}
return nil
}

1
internal/testdata/build/Test.abi vendored
View File

@ -1 +0,0 @@
[{"inputs":[],"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"address","name":"","type":"address"},{"indexed":false,"internalType":"uint256","name":"","type":"uint256"}],"name":"logPut","type":"event"},{"inputs":[{"internalType":"uint256","name":"value","type":"uint256"}],"name":"Put","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"close","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"data","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"}]

1
internal/testdata/build/Test.bin vendored
View File

@ -1 +0,0 @@
608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550610459806100606000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a14610050578063b90d3d0c1461006c575b600080fd5b61004e61009c565b005b61006a60048036038101906100659190610266565b610193565b005b610086600480360381019061008191906102f1565b610213565b604051610093919061032d565b60405180910390f35b60008054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161461012a576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610121906103cb565b60405180910390fd5b60008054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff166108fc479081150290604051600060405180830381858888f19350505050158015610190573d6000803e3d6000fd5b50565b7f370acc53a76362ca0f71a1b2e0c8b8ffbbc1ba9ff3166a1e2fa8445b4848626c33826040516101c49291906103fa565b60405180910390a180600160003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000208190555050565b60016020528060005260406000206000915090505481565b600080fd5b6000819050919050565b61024381610230565b811461024e57600080fd5b50565b6000813590506102608161023a565b92915050565b60006020828403121561027c5761027b61022b565b5b600061028a84828501610251565b91505092915050565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006102be82610293565b9050919050565b6102ce816102b3565b81146102d957600080fd5b50565b6000813590506102eb816102c5565b92915050565b6000602082840312156103075761030661022b565b5b6000610315848285016102dc565b91505092915050565b61032781610230565b82525050565b6000602082019050610342600083018461031e565b92915050565b600082825260208201905092915050565b7f4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f60008201527f6e2e000000000000000000000000000000000000000000000000000000000000602082015250565b60006103b5602283610348565b91506103c082610359565b604082019050919050565b600060208201905081810360008301526103e4816103a8565b9050919050565b6103f4816102b3565b82525050565b600060408201905061040f60008301856103eb565b61041c602083018461031e565b939250505056fea26469706673582212205c59738e694ff2e1f03f21fa2e7d943137128c0f43ae0e8175a589b32fa81a9a64736f6c63430008140033

10
internal/testdata/contract.go vendored
View File

@ -1,10 +0,0 @@
package testdata
import _ "embed"
var (
//go:embed build/Test.abi
TestContractABI string
//go:embed build/Test.bin
TestContractCode string
)

18
main.go
View File

@ -1,22 +1,6 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package main package main
import "github.com/cerc-io/ipld-eth-db-validator/v5/cmd" import "github.com/vulcanize/ipld-eth-db-validator/cmd"
func main() { func main() {
cmd.Execute() cmd.Execute()

157
pkg/prom/db_stats_collector.go
View File

@ -1,157 +0,0 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package prom
import (
"database/sql"
"github.com/prometheus/client_golang/prometheus"
)
// DBStatsGetter is an interface that gets sql.DBStats.
type DBStatsGetter interface {
Stats() sql.DBStats
}
// DBStatsCollector implements the prometheus.Collector interface.
type DBStatsCollector struct {
sg DBStatsGetter
// descriptions of exported metrics
maxOpenDesc *prometheus.Desc
openDesc *prometheus.Desc
inUseDesc *prometheus.Desc
idleDesc *prometheus.Desc
waitedForDesc *prometheus.Desc
blockedSecondsDesc *prometheus.Desc
closedMaxIdleDesc *prometheus.Desc
closedMaxLifetimeDesc *prometheus.Desc
}
// NewDBStatsCollector creates a new DBStatsCollector.
func NewDBStatsCollector(dbName string, sg DBStatsGetter) *DBStatsCollector {
labels := prometheus.Labels{"db_name": dbName}
return &DBStatsCollector{
sg: sg,
maxOpenDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "max_open"),
"Maximum number of open connections to the database.",
nil,
labels,
),
openDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "open"),
"The number of established connections both in use and idle.",
nil,
labels,
),
inUseDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "in_use"),
"The number of connections currently in use.",
nil,
labels,
),
idleDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "idle"),
"The number of idle connections.",
nil,
labels,
),
waitedForDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "waited_for"),
"The total number of connections waited for.",
nil,
labels,
),
blockedSecondsDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "blocked_seconds"),
"The total time blocked waiting for a new connection.",
nil,
labels,
),
closedMaxIdleDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "closed_max_idle"),
"The total number of connections closed due to SetMaxIdleConns.",
nil,
labels,
),
closedMaxLifetimeDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "closed_max_lifetime"),
"The total number of connections closed due to SetConnMaxLifetime.",
nil,
labels,
),
}
}
// Describe implements the prometheus.Collector interface.
func (c DBStatsCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.maxOpenDesc
ch <- c.openDesc
ch <- c.inUseDesc
ch <- c.idleDesc
ch <- c.waitedForDesc
ch <- c.blockedSecondsDesc
ch <- c.closedMaxIdleDesc
ch <- c.closedMaxLifetimeDesc
}
// Collect implements the prometheus.Collector interface.
func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) {
stats := c.sg.Stats()
ch <- prometheus.MustNewConstMetric(
c.maxOpenDesc,
prometheus.GaugeValue,
float64(stats.MaxOpenConnections),
)
ch <- prometheus.MustNewConstMetric(
c.openDesc,
prometheus.GaugeValue,
float64(stats.OpenConnections),
)
ch <- prometheus.MustNewConstMetric(
c.inUseDesc,
prometheus.GaugeValue,
float64(stats.InUse),
)
ch <- prometheus.MustNewConstMetric(
c.idleDesc,
prometheus.GaugeValue,
float64(stats.Idle),
)
ch <- prometheus.MustNewConstMetric(
c.waitedForDesc,
prometheus.CounterValue,
float64(stats.WaitCount),
)
ch <- prometheus.MustNewConstMetric(
c.blockedSecondsDesc,
prometheus.CounterValue,
stats.WaitDuration.Seconds(),
)
ch <- prometheus.MustNewConstMetric(
c.closedMaxIdleDesc,
prometheus.CounterValue,
float64(stats.MaxIdleClosed),
)
ch <- prometheus.MustNewConstMetric(
c.closedMaxLifetimeDesc,
prometheus.CounterValue,
float64(stats.MaxLifetimeClosed),
)
}

59
pkg/prom/prom.go
View File

@ -1,59 +0,0 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package prom
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
const (
namespace = "ipld_eth_state_snapshot"
connSubsystem = "connections"
statsSubsystem = "stats"
)
var (
metrics bool
lastValidatedBlock prometheus.Gauge
)
func Init() {
metrics = true
lastValidatedBlock = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "last_validated_block",
Help: "Last validated block number",
})
}
// RegisterDBCollector create metric collector for given connection
func RegisterDBCollector(name string, db DBStatsGetter) {
if metrics {
prometheus.Register(NewDBStatsCollector(name, db))
}
}
// SetLastValidatedBlock sets the last validated block number
func SetLastValidatedBlock(blockNumber float64) {
if metrics {
lastValidatedBlock.Set(blockNumber)
}
}

47
pkg/prom/serve.go
View File

@ -1,47 +0,0 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package prom
import (
"errors"
"net/http"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/sirupsen/logrus"
)
var errPromHTTP = errors.New("can't start http server for prometheus")
// Serve start listening http
func Serve(addr string) *http.Server {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
srv := http.Server{
Addr: addr,
Handler: mux,
}
go func() {
if err := srv.ListenAndServe(); err != nil {
logrus.
WithError(err).
WithField("module", "prom").
WithField("addr", addr).
Fatal(errPromHTTP)
}
}()
return &srv
}

170
pkg/validator/config.go
View File

@ -1,43 +1,70 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator package validator
import ( import (
"fmt" "fmt"
"math/big"
"time" "time"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/utils"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
) )
type Config struct { var (
DBConfig postgres.Config DATABASE_NAME = "DATABASE_NAME"
DBStats bool DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
)
ChainConfig *params.ChainConfig var IntegrationTestChainConfig = &params.ChainConfig{
// Used to trigger writing state diffs for gaps in the index ChainID: big.NewInt(99),
Client *rpc.Client HomesteadBlock: big.NewInt(0),
FromBlock, Trail uint64 EIP150Block: big.NewInt(0),
RetryInterval time.Duration EIP155Block: big.NewInt(0),
StateDiffMissingBlock bool EIP158Block: big.NewInt(0),
StateDiffTimeout time.Duration ByzantiumBlock: big.NewInt(0),
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
Clique: &params.CliqueConfig{
Period: 0,
Epoch: 30000,
},
}
var TestChainConfig = &params.ChainConfig{
ChainID: big.NewInt(1),
HomesteadBlock: big.NewInt(0),
EIP150Block: big.NewInt(0),
EIP155Block: big.NewInt(0),
EIP158Block: big.NewInt(0),
ByzantiumBlock: big.NewInt(0),
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
MuirGlacierBlock: big.NewInt(0),
BerlinBlock: big.NewInt(0),
LondonBlock: big.NewInt(6),
ArrowGlacierBlock: big.NewInt(0),
Ethash: new(params.EthashConfig),
}
type Config struct {
dbConfig postgres.Config
DB *sqlx.DB
ChainCfg *params.ChainConfig
BlockNum, Trail uint64
SleepInterval uint
} }
func NewConfig() (*Config, error) { func NewConfig() (*Config, error) {
@ -47,12 +74,16 @@ func NewConfig() (*Config, error) {
return nil, err return nil, err
} }
err = cfg.setupEth() cfg.BlockNum = viper.GetUint64("validate.block-height")
if err != nil { if cfg.BlockNum < 1 {
return nil, err return nil, fmt.Errorf("block height cannot be less the 1")
} }
err = cfg.setupValidator() cfg.Trail = viper.GetUint64("validate.trail")
cfg.SleepInterval = viper.GetUint("validate.sleepInterval")
chainConfigPath := viper.GetString("ethereum.chainConfig")
cfg.ChainCfg, err = statediff.LoadConfig(chainConfigPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -61,59 +92,32 @@ func NewConfig() (*Config, error) {
} }
func (c *Config) setupDB() error { func (c *Config) setupDB() error {
_ = viper.BindEnv("database.name", DATABASE_NAME)
_ = viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
_ = viper.BindEnv("database.port", DATABASE_PORT)
_ = viper.BindEnv("database.user", DATABASE_USER)
_ = viper.BindEnv("database.password", DATABASE_PASSWORD)
_ = viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
_ = viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
_ = viper.BindEnv("database.maxLifetime", DATABASE_MAX_CONN_LIFETIME)
// DB Config // DB Config
c.DBConfig.DatabaseName = viper.GetString("database.name") c.dbConfig.DatabaseName = viper.GetString("database.name")
c.DBConfig.Hostname = viper.GetString("database.hostname") c.dbConfig.Hostname = viper.GetString("database.hostname")
c.DBConfig.Port = viper.GetInt("database.port") c.dbConfig.Port = viper.GetInt("database.port")
c.DBConfig.Username = viper.GetString("database.user") c.dbConfig.Username = viper.GetString("database.user")
c.DBConfig.Password = viper.GetString("database.password") c.dbConfig.Password = viper.GetString("database.password")
c.DBConfig.MaxIdle = viper.GetInt("database.maxIdle") c.dbConfig.MaxIdle = viper.GetInt("database.maxIdle")
c.DBConfig.MaxConns = viper.GetInt("database.maxOpen") c.dbConfig.MaxConns = viper.GetInt("database.maxOpen")
c.DBConfig.MaxConnLifetime = viper.GetDuration("database.maxLifetime") c.dbConfig.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime"))
c.DBStats = viper.GetBool("prom.dbStats") // Create DB
db, err := shared.NewDB(c.dbConfig.DbConnectionString(), c.dbConfig)
if err != nil {
return fmt.Errorf("failed to create config: %w", err)
}
c.DB = db
return nil return nil
} }
func (c *Config) setupEth() error {
var err error
chainConfigPath := viper.GetString("ethereum.chainConfig")
if chainConfigPath != "" {
c.ChainConfig, err = utils.LoadConfig(chainConfigPath)
} else {
// read chainID if chain config path not provided
chainID := viper.GetUint64("ethereum.chainID")
c.ChainConfig, err = utils.ChainConfig(chainID)
}
if err != nil {
return err
}
// setup a statediffing client
ethHTTP := viper.GetString("ethereum.httpPath")
if ethHTTP != "" {
ethHTTPEndpoint := fmt.Sprintf("http://%s", ethHTTP)
c.Client, err = rpc.Dial(ethHTTPEndpoint)
}
return err
}
func (c *Config) setupValidator() error {
var err error
c.FromBlock = viper.GetUint64("validate.fromBlock")
if c.FromBlock < 1 {
return fmt.Errorf("starting block height cannot be less than 1")
}
c.Trail = viper.GetUint64("validate.trail")
c.RetryInterval = viper.GetDuration("validate.retryInterval")
c.StateDiffMissingBlock = viper.GetBool("validate.stateDiffMissingBlock")
if c.StateDiffMissingBlock {
c.StateDiffTimeout = viper.GetDuration("validate.stateDiffTimeout")
}
return err
}

125
pkg/validator/database.go
View File

@ -1,19 +1,3 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator package validator
import ( import (
@ -25,19 +9,120 @@ import (
var errNotSupported = errors.New("this operation is not supported") var errNotSupported = errors.New("this operation is not supported")
type database struct { type database struct {
ethdb.Database ethDB ethdb.Database
} }
func newDatabase(db ethdb.Database) *database { func newDatabase(db ethdb.Database) *database {
return &database{ return &database{
Database: db, ethDB: db,
} }
} }
func (d *database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
return d.ethDB.NewIterator(prefix, start)
}
func (d *database) Has(key []byte) (bool, error) {
return d.ethDB.Has(key)
}
func (d *database) Get(key []byte) ([]byte, error) {
return d.ethDB.Get(key)
}
func (d *database) Put(key []byte, value []byte) error { func (d *database) Put(key []byte, value []byte) error {
return errNotSupported return nil
} }
func (d *database) Delete(key []byte) error { func (d *database) Delete(key []byte) error {
return errNotSupported return nil
}
func (d *database) Stat(property string) (string, error) {
return d.ethDB.Stat(property)
}
func (d *database) Compact(start []byte, limit []byte) error {
return d.ethDB.Compact(start, limit)
}
// HasAncient returns an error as we don't have a backing chain freezer.
func (d *database) HasAncient(kind string, number uint64) (bool, error) {
return d.ethDB.HasAncient(kind, number)
}
// Ancient returns an error as we don't have a backing chain freezer.
func (d *database) Ancient(kind string, number uint64) ([]byte, error) {
return d.ethDB.Ancient(kind, number)
}
// AncientRange returns an error as we don't have a backing chain freezer.
func (d *database) AncientRange(kind string, start, max, maxByteSize uint64) ([][]byte, error) {
return d.ethDB.AncientRange(kind, start, max, maxByteSize)
}
// Ancients returns an error as we don't have a backing chain freezer.
func (d *database) Ancients() (uint64, error) {
return d.ethDB.Ancients()
}
// AncientSize returns an error as we don't have a backing chain freezer.
func (d *database) AncientSize(kind string) (uint64, error) {
return d.ethDB.AncientSize(kind)
}
// Tail returns the number of first stored item in the freezer.
func (d *database) Tail() (uint64, error) {
return d.Tail()
}
// ModifyAncients is not supported.
func (d *database) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (int64, error) {
return 0, nil
}
// TruncateHead discards all but the first n ancient data from the ancient store.
func (d *database) TruncateHead(n uint64) error {
return nil
}
// TruncateTail discards the first n ancient data from the ancient store.
func (d *database) TruncateTail(n uint64) error {
return nil
}
func (d *database) Sync() error {
return d.ethDB.Sync()
}
// MigrateTable processes and migrates entries of a given table to a new format.
func (d *database) MigrateTable(string, func([]byte) ([]byte, error)) error {
return nil
}
func (d *database) NewBatch() ethdb.Batch {
return d.ethDB.NewBatch()
}
// NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
func (d *database) NewBatchWithSize(size int) ethdb.Batch {
return d.ethDB.NewBatchWithSize(size)
}
func (d *database) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
return d.ethDB.ReadAncients(fn)
}
func (d *database) Close() error {
return d.ethDB.Close()
}
// NewSnapshot creates a database snapshot based on the current state.
func (d *database) NewSnapshot() (ethdb.Snapshot, error) {
return d.NewSnapshot()
}
// NewSnapshot creates a database snapshot based on the current state.
func (d *database) AncientDatadir() (string, error) {
return "", errNotSupported
} }

13
pkg/validator/errors.go
View File

@ -1,13 +0,0 @@
package validator
import (
"fmt"
)
type ChainNotSyncedError struct {
Head uint64
}
func (e *ChainNotSyncedError) Error() string {
return fmt.Sprintf("chain not synced (current head: %d)", e.Head)
}

124
pkg/validator/ref_integrity.go
View File

@ -1,19 +1,3 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator package validator
import ( import (
@ -22,44 +6,50 @@ import (
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
) )
var (
ReferentialIntegrityErr = "referential integrity check failed at block %d, entry for %s not found"
EntryNotFoundErr = "entry for %s not found"
)
// ValidateReferentialIntegrity validates referential integrity at the given height // ValidateReferentialIntegrity validates referential integrity at the given height
func ValidateReferentialIntegrity(tx *sqlx.Tx, blockNumber uint64) error { func ValidateReferentialIntegrity(db *sqlx.DB, blockNumber uint64) error {
err := ValidateHeaderCIDsRef(tx, blockNumber)
err := ValidateHeaderCIDsRef(db, blockNumber)
if err != nil { if err != nil {
return err return err
} }
err = ValidateUncleCIDsRef(tx, blockNumber) err = ValidateUncleCIDsRef(db, blockNumber)
if err != nil { if err != nil {
return err return err
} }
err = ValidateTransactionCIDsRef(tx, blockNumber) err = ValidateTransactionCIDsRef(db, blockNumber)
if err != nil { if err != nil {
return err return err
} }
err = ValidateReceiptCIDsRef(tx, blockNumber) err = ValidateReceiptCIDsRef(db, blockNumber)
if err != nil { if err != nil {
return err return err
} }
err = ValidateStateCIDsRef(tx, blockNumber) err = ValidateStateCIDsRef(db, blockNumber)
if err != nil { if err != nil {
return err return err
} }
err = ValidateStorageCIDsRef(tx, blockNumber) err = ValidateStorageCIDsRef(db, blockNumber)
if err != nil { if err != nil {
return err return err
} }
err = ValidateLogCIDsRef(tx, blockNumber) err = ValidateStateAccountsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateAccessListElementsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateLogCIDsRef(db, blockNumber)
if err != nil { if err != nil {
return err return err
} }
@ -68,8 +58,8 @@ func ValidateReferentialIntegrity(tx *sqlx.Tx, blockNumber uint64) error {
} }
// ValidateHeaderCIDsRef does a reference integrity check on references in eth.header_cids table // ValidateHeaderCIDsRef does a reference integrity check on references in eth.header_cids table
func ValidateHeaderCIDsRef(tx *sqlx.Tx, blockNumber uint64) error { func ValidateHeaderCIDsRef(db *sqlx.DB, blockNumber uint64) error {
err := ValidateIPFSBlocks(tx, blockNumber, "eth.header_cids", "cid") err := ValidateIPFSBlocks(db, blockNumber, "eth.header_cids", "mh_key")
if err != nil { if err != nil {
return err return err
} }
@ -78,9 +68,9 @@ func ValidateHeaderCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
} }
// ValidateUncleCIDsRef does a reference integrity check on references in eth.uncle_cids table // ValidateUncleCIDsRef does a reference integrity check on references in eth.uncle_cids table
func ValidateUncleCIDsRef(tx *sqlx.Tx, blockNumber uint64) error { func ValidateUncleCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool var exists bool
err := tx.Get(&exists, UncleCIDsRefHeaderCIDs, blockNumber) err := db.Get(&exists, UncleCIDsRefHeaderCIDs, blockNumber)
if err != nil { if err != nil {
return err return err
} }
@ -88,7 +78,7 @@ func ValidateUncleCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids") return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
} }
err = ValidateIPFSBlocks(tx, blockNumber, "eth.uncle_cids", "cid") err = ValidateIPFSBlocks(db, blockNumber, "eth.uncle_cids", "mh_key")
if err != nil { if err != nil {
return err return err
} }
@ -97,9 +87,9 @@ func ValidateUncleCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
} }
// ValidateTransactionCIDsRef does a reference integrity check on references in eth.header_cids table // ValidateTransactionCIDsRef does a reference integrity check on references in eth.header_cids table
func ValidateTransactionCIDsRef(tx *sqlx.Tx, blockNumber uint64) error { func ValidateTransactionCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool var exists bool
err := tx.Get(&exists, TransactionCIDsRefHeaderCIDs, blockNumber) err := db.Get(&exists, TransactionCIDsRefHeaderCIDs, blockNumber)
if err != nil { if err != nil {
return err return err
} }
@ -107,7 +97,7 @@ func ValidateTransactionCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids") return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
} }
err = ValidateIPFSBlocks(tx, blockNumber, "eth.transaction_cids", "cid") err = ValidateIPFSBlocks(db, blockNumber, "eth.transaction_cids", "mh_key")
if err != nil { if err != nil {
return err return err
} }
@ -116,9 +106,9 @@ func ValidateTransactionCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
} }
// ValidateReceiptCIDsRef does a reference integrity check on references in eth.receipt_cids table // ValidateReceiptCIDsRef does a reference integrity check on references in eth.receipt_cids table
func ValidateReceiptCIDsRef(tx *sqlx.Tx, blockNumber uint64) error { func ValidateReceiptCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool var exists bool
err := tx.Get(&exists, ReceiptCIDsRefTransactionCIDs, blockNumber) err := db.Get(&exists, ReceiptCIDsRefTransactionCIDs, blockNumber)
if err != nil { if err != nil {
return err return err
} }
@ -126,7 +116,7 @@ func ValidateReceiptCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids") return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
} }
err = ValidateIPFSBlocks(tx, blockNumber, "eth.receipt_cids", "cid") err = ValidateIPFSBlocks(db, blockNumber, "eth.receipt_cids", "leaf_mh_key")
if err != nil { if err != nil {
return err return err
} }
@ -135,9 +125,9 @@ func ValidateReceiptCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
} }
// ValidateStateCIDsRef does a reference integrity check on references in eth.state_cids table // ValidateStateCIDsRef does a reference integrity check on references in eth.state_cids table
func ValidateStateCIDsRef(tx *sqlx.Tx, blockNumber uint64) error { func ValidateStateCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool var exists bool
err := tx.Get(&exists, StateCIDsRefHeaderCIDs, blockNumber) err := db.Get(&exists, StateCIDsRefHeaderCIDs, blockNumber)
if err != nil { if err != nil {
return err return err
} }
@ -145,7 +135,7 @@ func ValidateStateCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids") return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
} }
err = ValidateIPFSBlocks(tx, blockNumber, "eth.state_cids", "cid") err = ValidateIPFSBlocks(db, blockNumber, "eth.state_cids", "mh_key")
if err != nil { if err != nil {
return err return err
} }
@ -154,9 +144,9 @@ func ValidateStateCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
} }
// ValidateStorageCIDsRef does a reference integrity check on references in eth.storage_cids table // ValidateStorageCIDsRef does a reference integrity check on references in eth.storage_cids table
func ValidateStorageCIDsRef(tx *sqlx.Tx, blockNumber uint64) error { func ValidateStorageCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool var exists bool
err := tx.Get(&exists, StorageCIDsRefStateCIDs, blockNumber) err := db.Get(&exists, StorageCIDsRefStateCIDs, blockNumber)
if err != nil { if err != nil {
return err return err
} }
@ -164,7 +154,7 @@ func ValidateStorageCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids") return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
} }
err = ValidateIPFSBlocks(tx, blockNumber, "eth.storage_cids", "cid") err = ValidateIPFSBlocks(db, blockNumber, "eth.storage_cids", "mh_key")
if err != nil { if err != nil {
return err return err
} }
@ -172,10 +162,38 @@ func ValidateStorageCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return nil return nil
} }
// ValidateLogCIDsRef does a reference integrity check on references in eth.log_cids table // ValidateStateAccountsRef does a reference integrity check on references in eth.state_accounts table
func ValidateLogCIDsRef(tx *sqlx.Tx, blockNumber uint64) error { func ValidateStateAccountsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool var exists bool
err := tx.Get(&exists, LogCIDsRefReceiptCIDs, blockNumber) err := db.Get(&exists, StateAccountsRefStateCIDs, blockNumber)
if err != nil {
return err
}
if exists {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
}
return nil
}
// ValidateAccessListElementsRef does a reference integrity check on references in eth.access_list_elements table
func ValidateAccessListElementsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := db.Get(&exists, AccessListElementsRefTransactionCIDs, blockNumber)
if err != nil {
return err
}
if exists {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
}
return nil
}
// ValidateLogCIDsRef does a reference integrity check on references in eth.log_cids table
func ValidateLogCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := db.Get(&exists, LogCIDsRefReceiptCIDs, blockNumber)
if err != nil { if err != nil {
return err return err
} }
@ -183,7 +201,7 @@ func ValidateLogCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.receipt_cids") return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.receipt_cids")
} }
err = ValidateIPFSBlocks(tx, blockNumber, "eth.log_cids", "cid") err = ValidateIPFSBlocks(db, blockNumber, "eth.log_cids", "leaf_mh_key")
if err != nil { if err != nil {
return err return err
} }
@ -192,14 +210,14 @@ func ValidateLogCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
} }
// ValidateIPFSBlocks does a reference integrity check between the given CID table and IPFS blocks table on MHKey and block number // ValidateIPFSBlocks does a reference integrity check between the given CID table and IPFS blocks table on MHKey and block number
func ValidateIPFSBlocks(tx *sqlx.Tx, blockNumber uint64, CIDTable string, CIDField string) error { func ValidateIPFSBlocks(db *sqlx.DB, blockNumber uint64, CIDTable string, mhKeyField string) error {
var exists bool var exists bool
err := tx.Get(&exists, fmt.Sprintf(CIDsRefIPLDBlocks, CIDTable, CIDField), blockNumber) err := db.Get(&exists, fmt.Sprintf(CIDsRefIPLDBlocks, CIDTable, mhKeyField), blockNumber)
if err != nil { if err != nil {
return err return err
} }
if exists { if exists {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "ipld.blocks") return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "public.blocks")
} }
return nil return nil

47
pkg/validator/ref_integrity_queries.go
View File

@ -1,19 +1,3 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator package validator
// Queries to validate referential integrity in the indexed data: // Queries to validate referential integrity in the indexed data:
@ -26,7 +10,7 @@ const (
CIDsRefIPLDBlocks = `SELECT EXISTS ( CIDsRefIPLDBlocks = `SELECT EXISTS (
SELECT * SELECT *
FROM %[1]s FROM %[1]s
LEFT JOIN ipld.blocks ON ( LEFT JOIN public.blocks ON (
%[1]s.%[2]s = blocks.key %[1]s.%[2]s = blocks.key
AND %[1]s.block_number = blocks.block_number AND %[1]s.block_number = blocks.block_number
) )
@ -88,13 +72,38 @@ const (
SELECT * SELECT *
FROM eth.storage_cids FROM eth.storage_cids
LEFT JOIN eth.state_cids ON ( LEFT JOIN eth.state_cids ON (
storage_cids.state_leaf_key = state_cids.state_leaf_key storage_cids.state_path = state_cids.state_path
AND storage_cids.header_id = state_cids.header_id AND storage_cids.header_id = state_cids.header_id
AND storage_cids.block_number = state_cids.block_number AND storage_cids.block_number = state_cids.block_number
) )
WHERE WHERE
storage_cids.block_number = $1 storage_cids.block_number = $1
AND state_cids.state_leaf_key IS NULL AND state_cids.state_path IS NULL
)`
StateAccountsRefStateCIDs = `SELECT EXISTS (
SELECT *
FROM eth.state_accounts
LEFT JOIN eth.state_cids ON (
state_accounts.state_path = state_cids.state_path
AND state_accounts.header_id = state_cids.header_id
AND state_accounts.block_number = state_cids.block_number
)
WHERE
state_accounts.block_number = $1
AND state_cids.state_path IS NULL
)`
AccessListElementsRefTransactionCIDs = `SELECT EXISTS (
SELECT *
FROM eth.access_list_elements
LEFT JOIN eth.transaction_cids ON (
access_list_elements.tx_id = transaction_cids.tx_hash
AND access_list_elements.block_number = transaction_cids.block_number
)
WHERE
access_list_elements.block_number = $1
AND transaction_cids.tx_hash IS NULL
)` )`
LogCIDsRefReceiptCIDs = `SELECT EXISTS ( LogCIDsRefReceiptCIDs = `SELECT EXISTS (

294
pkg/validator/ref_integrity_test.go
View File

@ -3,256 +3,332 @@ package validator_test
import ( import (
"context" "context"
"fmt" "fmt"
"math/big"
"testing"
indexer_helpers "github.com/cerc-io/plugeth-statediff/indexer/test_helpers" "github.com/ethereum/go-ethereum/params"
helpers "github.com/cerc-io/plugeth-statediff/test_helpers" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/cerc-io/ipld-eth-db-validator/v5/internal/chaingen" "github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/validator" "github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
) )
func TestRefIntegrity(t *testing.T) { var _ = Describe("RefIntegrity", func() {
RegisterFailHandler(Fail)
RunSpecs(t, "ETH IPLD validator ref integrity suite test")
}
var _ = Describe("referential integrity", Ordered, func() {
var ( var (
db *sqlx.DB ctx = context.Background()
tx *sqlx.Tx
checkedBlock *types.Block // Generated block of interest db *sqlx.DB
diffIndexer interfaces.StateDiffIndexer
) )
BeforeAll(func() {
var (
blocks []*types.Block
receipts []types.Receipts
chain *core.BlockChain
chainConfig = TestChainConfig
mockTD = big.NewInt(1337)
testdb = rawdb.NewMemoryDatabase()
)
gen := chaingen.DefaultGenContext(chainConfig, testdb) BeforeEach(func() {
gen.AddFunction(func(i int, block *core.BlockGen) { db = shared.SetupDB()
if i >= 2 { diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
uncle := &types.Header{
Number: big.NewInt(int64(i - 1)),
Root: common.HexToHash("0x1"),
TxHash: common.HexToHash("0x1"),
ReceiptHash: common.HexToHash("0x1"),
ParentHash: block.PrevBlock(i - 1).Hash(),
}
block.AddUncle(uncle)
}
})
blocks, receipts, chain = gen.MakeChain(5)
indexer, err := helpers.NewIndexer(context.Background(), chainConfig, gen.Genesis.Hash(), TestDBConfig)
Expect(err).ToNot(HaveOccurred())
helpers.IndexChain(indexer, helpers.IndexChainParams{
StateCache: chain.StateCache(),
Blocks: blocks,
Receipts: receipts,
TotalDifficulty: mockTD,
})
checkedBlock = blocks[5]
db = SetupDB()
}) })
AfterAll(func() { Expect(indexer_helpers.ClearSqlxDB(db)).ToNot(HaveOccurred()) })
BeforeEach(func() { tx = db.MustBegin() })
AfterEach(func() { AfterEach(func() {
tx.Rollback() shared.TearDownDB(db)
}) })
Describe("ValidateHeaderCIDsRef", func() { Describe("ValidateHeaderCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of header_cids table", func() { It("Validates referential integrity of header_cids table", func() {
err := validator.ValidateHeaderCIDsRef(tx, checkedBlock.NumberU64()) err := validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
It("Throws an error if corresponding header IPFS block entry not found", func() { It("Throws an error if corresponding header IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks") err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateHeaderCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
}) })
}) })
Describe("ValidateUncleCIDsRef", func() { Describe("ValidateUncleCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of uncle_cids table", func() { It("Validates referential integrity of uncle_cids table", func() {
err := validator.ValidateUncleCIDsRef(tx, checkedBlock.NumberU64()) err := validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
It("Throws an error if corresponding header_cid entry not found", func() { It("Throws an error if corresponding header_cid entry not found", func() {
err := deleteEntriesFrom(tx, "eth.header_cids") err := deleteEntriesFrom(db, "eth.header_cids")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateUncleCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
}) })
It("Throws an error if corresponding uncle IPFS block entry not found", func() { It("Throws an error if corresponding uncle IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks") err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateUncleCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
}) })
}) })
Describe("ValidateTransactionCIDsRef", func() { Describe("ValidateTransactionCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of transaction_cids table", func() { It("Validates referential integrity of transaction_cids table", func() {
err := validator.ValidateTransactionCIDsRef(tx, checkedBlock.NumberU64()) err := validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
It("Throws an error if corresponding header_cid entry not found", func() { It("Throws an error if corresponding header_cid entry not found", func() {
err := deleteEntriesFrom(tx, "eth.header_cids") err := deleteEntriesFrom(db, "eth.header_cids")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateTransactionCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
}) })
It("Throws an error if corresponding transaction IPFS block entry not found", func() { It("Throws an error if corresponding transaction IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks") err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateTransactionCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
}) })
}) })
Describe("ValidateReceiptCIDsRef", func() { Describe("ValidateReceiptCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of receipt_cids table", func() { It("Validates referential integrity of receipt_cids table", func() {
err := validator.ValidateReceiptCIDsRef(tx, checkedBlock.NumberU64()) err := validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
It("Throws an error if corresponding transaction_cids entry not found", func() { It("Throws an error if corresponding transaction_cids entry not found", func() {
err := deleteEntriesFrom(tx, "eth.transaction_cids") err := deleteEntriesFrom(db, "eth.transaction_cids")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateReceiptCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
}) })
It("Throws an error if corresponding receipt IPFS block entry not found", func() { It("Throws an error if corresponding receipt IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks") err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateReceiptCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
}) })
}) })
Describe("ValidateStateCIDsRef", func() { Describe("ValidateStateCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of state_cids table", func() { It("Validates referential integrity of state_cids table", func() {
err := validator.ValidateStateCIDsRef(tx, checkedBlock.NumberU64()) err := validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
It("Throws an error if corresponding header_cids entry not found", func() { It("Throws an error if corresponding header_cids entry not found", func() {
err := deleteEntriesFrom(tx, "eth.header_cids") err := deleteEntriesFrom(db, "eth.header_cids")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStateCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
}) })
It("Throws an error if corresponding state IPFS block entry not found", func() { It("Throws an error if corresponding state IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks") err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStateCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
}) })
}) })
Describe("ValidateStorageCIDsRef", func() { Describe("ValidateStorageCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of storage_cids table", func() { It("Validates referential integrity of storage_cids table", func() {
err := validator.ValidateStorageCIDsRef(tx, checkedBlock.NumberU64()) err := validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
It("Throws an error if corresponding state_cids entry not found", func() { It("Throws an error if corresponding state_cids entry not found", func() {
err := deleteEntriesFrom(tx, "eth.state_cids") err := deleteEntriesFrom(db, "eth.state_cids")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStorageCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
}) })
It("Throws an error if corresponding storage IPFS block entry not found", func() { It("Throws an error if corresponding storage IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks") err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStorageCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
})
})
Describe("ValidateStateAccountsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of state_accounts table", func() {
err := validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding state_cids entry not found", func() {
err := deleteEntriesFrom(db, "eth.state_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
})
})
Describe("ValidateAccessListElementsRef", func() {
BeforeEach(func() {
indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, mocks.TestConfig, test_helpers.Genesis.Hash())
tx, err := indexAndPublisher.PushBlock(mocks.MockBlock, mocks.MockReceipts, mocks.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of access_list_elements table", func() {
err := validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding transaction_cids entry not found", func() {
err := deleteEntriesFrom(db, "eth.transaction_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
}) })
}) })
Describe("ValidateLogCIDsRef", func() { Describe("ValidateLogCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of log_cids table", func() { It("Validates referential integrity of log_cids table", func() {
err := validator.ValidateLogCIDsRef(tx, checkedBlock.NumberU64()) err := validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
It("Throws an error if corresponding receipt_cids entry not found", func() { It("Throws an error if corresponding receipt_cids entry not found", func() {
err := deleteEntriesFrom(tx, "eth.receipt_cids") err := deleteEntriesFrom(db, "eth.receipt_cids")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateLogCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.receipt_cids")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.receipt_cids"))
}) })
It("Throws an error if corresponding log IPFS block entry not found", func() { It("Throws an error if corresponding log IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks") err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
err = validator.ValidateLogCIDsRef(tx, checkedBlock.NumberU64()) err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks")) Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
}) })
}) })
Describe("ValidateReferentialIntegrity", func() {
It("Validates referential integrity of full chain", func() {
for i := uint64(startBlock); i <= chainLength; i++ {
err := validator.ValidateReferentialIntegrity(tx, i)
Expect(err).ToNot(HaveOccurred())
}
})
})
}) })
func deleteEntriesFrom(tx *sqlx.Tx, tableName string) error { func deleteEntriesFrom(db *sqlx.DB, tableName string) error {
pgStr := "TRUNCATE %s" pgStr := "DELETE FROM %s"
_, err := tx.Exec(fmt.Sprintf(pgStr, tableName)) _, err := db.Exec(fmt.Sprintf(pgStr, tableName))
return err return err
} }

46
pkg/validator/util_test.go
View File

@ -1,46 +0,0 @@
package validator_test
import (
"context"
"math/big"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/params"
"github.com/jmoiron/sqlx"
)
var TestChainConfig = &params.ChainConfig{
ChainID: big.NewInt(1),
HomesteadBlock: big.NewInt(0),
EIP150Block: big.NewInt(0),
EIP155Block: big.NewInt(0),
EIP158Block: big.NewInt(0),
ByzantiumBlock: big.NewInt(0),
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
MuirGlacierBlock: big.NewInt(0),
BerlinBlock: big.NewInt(0),
LondonBlock: big.NewInt(6),
ArrowGlacierBlock: big.NewInt(0),
GrayGlacierBlock: big.NewInt(0),
Ethash: new(params.EthashConfig),
}
var TestDBConfig postgres.Config
func init() {
var err error
TestDBConfig, err = postgres.TestConfig.WithEnv()
if err != nil {
panic(err)
}
}
func SetupDB() *sqlx.DB {
db, err := postgres.ConnectSQLX(context.Background(), TestDBConfig)
if err != nil {
panic(err)
}
return db
}

409
pkg/validator/validator.go
View File

@ -1,211 +1,193 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator package validator
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"math/big" "math/big"
"sync" "sync"
"time" "time"
statediff "github.com/cerc-io/plugeth-statediff"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/clique"
"github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/holiman/uint256"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
ipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0" ipfsethdb "github.com/vulcanize/ipfs-ethdb/v4/postgres"
ipldeth "github.com/cerc-io/ipld-eth-server/v5/pkg/eth" ipldEth "github.com/vulcanize/ipld-eth-server/v4/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v5/pkg/shared" ethServerShared "github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
ipldstate "github.com/cerc-io/ipld-eth-statedb/trie_by_cid/state"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/prom"
) )
var ( var (
big8 = big.NewInt(8) big8 = big.NewInt(8)
big32 = big.NewInt(32) big32 = big.NewInt(32)
ReferentialIntegrityErr = "referential integrity check failed at block %d, entry for %s not found"
EntryNotFoundErr = "entry for %s not found"
) )
type Service struct { type service struct {
db *sqlx.DB db *sqlx.DB
blockNum, trail uint64
chainConfig *params.ChainConfig sleepInterval uint
ethClient *rpc.Client logger *log.Logger
blockNum, trail uint64 chainCfg *params.ChainConfig
retryInterval time.Duration quitChan chan bool
stateDiffMissingBlock bool progressChan chan uint64
stateDiffTimeout time.Duration
quitChan chan bool
progressChan chan<- uint64
} }
func NewService(cfg *Config, progressChan chan<- uint64) (*Service, error) { func NewService(cfg *Config, progressChan chan uint64) *service {
db, err := postgres.ConnectSQLX(context.Background(), cfg.DBConfig) return &service{
if err != nil { db: cfg.DB,
return nil, err blockNum: cfg.BlockNum,
trail: cfg.Trail,
sleepInterval: cfg.SleepInterval,
logger: log.New(),
chainCfg: cfg.ChainCfg,
quitChan: make(chan bool),
progressChan: progressChan,
} }
// Enable DB stats }
if cfg.DBStats {
prom.RegisterDBCollector(cfg.DBConfig.DatabaseName, db) func NewEthBackend(db *sqlx.DB, c *ipldEth.Config) (*ipldEth.Backend, error) {
gcc := c.GroupCacheConfig
groupName := gcc.StateDB.Name
if groupName == "" {
groupName = ipldEth.StateDBGroupCacheName
} }
return &Service{ r := ipldEth.NewCIDRetriever(db)
db: db, ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{
chainConfig: cfg.ChainConfig, Name: groupName,
ethClient: cfg.Client, Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024,
blockNum: cfg.FromBlock, ExpiryDuration: time.Minute * time.Duration(gcc.StateDB.CacheExpiryInMins),
trail: cfg.Trail, })
retryInterval: cfg.RetryInterval,
stateDiffMissingBlock: cfg.StateDiffMissingBlock, // Read only wrapper around ipfs-ethdb eth.Database implementation
stateDiffTimeout: cfg.StateDiffTimeout, customEthDB := newDatabase(ethDB)
quitChan: make(chan bool),
progressChan: progressChan, return &ipldEth.Backend{
DB: db,
Retriever: r,
Fetcher: ipldEth.NewIPLDFetcher(db),
IPLDRetriever: ipldEth.NewIPLDRetriever(db),
EthDB: customEthDB,
StateDatabase: state.NewDatabase(customEthDB),
Config: c,
}, nil }, nil
} }
// Start is used to begin the service // Start is used to begin the service
func (s *Service) Start(ctx context.Context, wg *sync.WaitGroup) { func (s *service) Start(ctx context.Context, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
api, err := EthAPI(ctx, s.db, s.chainConfig) api, err := EthAPI(ctx, s.db, s.chainCfg)
if err != nil { if err != nil {
log.Fatal(err) s.logger.Fatal(err)
return return
} }
nextBlockNum := s.blockNum idxBlockNum := s.blockNum
var delay time.Duration
for { for {
select { select {
case <-s.quitChan: case <-s.quitChan:
log.Info("stopping ipld-eth-db-validator process") s.logger.Infof("last validated block %v", idxBlockNum-1)
if s.progressChan != nil { if s.progressChan != nil {
close(s.progressChan) close(s.progressChan)
} }
if err := api.B.Close(); err != nil {
log.Errorf("error closing backend: %s", err)
}
return return
case <-time.After(delay): default:
err := s.Validate(ctx, api, nextBlockNum) idxBlockNum, err = s.Validate(ctx, api, idxBlockNum)
// If chain is not synced, wait for trail to catch up before trying again
if notsynced, ok := err.(*ChainNotSyncedError); ok {
delay = s.retryInterval
log.Infof("waiting %v for chain to advance to block %d (head is at %d)",
delay, nextBlockNum+s.trail, notsynced.Head)
continue
}
if err != nil { if err != nil {
log.Fatal(err) s.logger.Infof("last validated block %v", idxBlockNum-1)
s.logger.Fatal(err)
return return
} }
prom.SetLastValidatedBlock(float64(nextBlockNum))
nextBlockNum++
delay = 0
} }
} }
} }
// Stop is used to gracefully stop the service // Stop is used to gracefully stop the service
func (s *Service) Stop() { func (s *service) Stop() {
s.logger.Info("stopping ipld-eth-db-validator process")
close(s.quitChan) close(s.quitChan)
} }
func (s *Service) Validate(ctx context.Context, api *ipldeth.PublicEthAPI, idxBlockNum uint64) error { func (s *service) Validate(ctx context.Context, api *ipldEth.PublicEthAPI, idxBlockNum uint64) (uint64, error) {
log.Debugf("validating block %d", idxBlockNum)
headBlockNum, err := fetchHeadBlockNumber(ctx, api) headBlockNum, err := fetchHeadBlockNumber(ctx, api)
if err != nil { if err != nil {
return err return idxBlockNum, err
} }
// Check if block at requested height can be validated // Check if it block at height idxBlockNum can be validated
if idxBlockNum+s.trail > headBlockNum { if idxBlockNum <= headBlockNum-s.trail {
return &ChainNotSyncedError{headBlockNum} err = ValidateBlock(ctx, api, idxBlockNum)
if err != nil {
s.logger.Errorf("failed to verify state root at block %d", idxBlockNum)
return idxBlockNum, err
}
s.logger.Infof("state root verified for block %d", idxBlockNum)
err = ValidateReferentialIntegrity(s.db, idxBlockNum)
if err != nil {
s.logger.Errorf("failed to verify referential integrity at block %d", idxBlockNum)
return idxBlockNum, err
}
s.logger.Infof("referential integrity verified for block %d", idxBlockNum)
if s.progressChan != nil {
s.progressChan <- idxBlockNum
}
idxBlockNum++
} else {
// Sleep / wait for head to move ahead
time.Sleep(time.Second * time.Duration(s.sleepInterval))
} }
blockToBeValidated, err := api.B.BlockByNumber(ctx, rpc.BlockNumber(idxBlockNum)) return idxBlockNum, nil
if err != nil {
log.Errorf("failed to fetch block at height %d", idxBlockNum)
return err
}
// Make a writeStateDiffAt call if block not found in the db
if blockToBeValidated == nil {
return s.writeStateDiffAt(idxBlockNum)
}
err = ValidateBlock(blockToBeValidated, api.B, idxBlockNum)
if err != nil {
log.Errorf("failed to verify state root at block %d", idxBlockNum)
return err
}
log.Infof("state root verified for block %d", idxBlockNum)
tx := s.db.MustBegin()
defer tx.Rollback()
err = ValidateReferentialIntegrity(tx, idxBlockNum)
if err != nil {
log.Errorf("failed to verify referential integrity at block %d", idxBlockNum)
return err
}
log.Infof("referential integrity verified for block %d", idxBlockNum)
if s.progressChan != nil {
s.progressChan <- idxBlockNum
}
return nil
} }
// ValidateBlock validates block at the given height // ValidateBlock validates block at the given height
func ValidateBlock(blockToBeValidated *types.Block, b *ipldeth.Backend, blockNumber uint64) error { func ValidateBlock(ctx context.Context, api *ipldEth.PublicEthAPI, blockNumber uint64) error {
state, err := applyTransactions(blockToBeValidated, b) blockToBeValidated, err := api.B.BlockByNumber(ctx, rpc.BlockNumber(blockNumber))
if err != nil { if err != nil {
return err return err
} }
blockStateRoot := blockToBeValidated.Header().Root stateDB, err := applyTransaction(blockToBeValidated, api.B)
dbStateRoot := state.IntermediateRoot(true) if err != nil {
return err
}
blockStateRoot := blockToBeValidated.Header().Root.String()
dbStateRoot := stateDB.IntermediateRoot(true).String()
if blockStateRoot != dbStateRoot { if blockStateRoot != dbStateRoot {
return fmt.Errorf("state roots do not match at block %d", blockNumber) return fmt.Errorf("state roots do not match at block %d", blockNumber)
} }
return nil return nil
} }
func EthAPI(ctx context.Context, db *sqlx.DB, chainCfg *params.ChainConfig) (*ipldeth.PublicEthAPI, error) { func EthAPI(ctx context.Context, db *sqlx.DB, chainCfg *params.ChainConfig) (*ipldEth.PublicEthAPI, error) {
// TODO: decide network for custom chainConfig. // TODO: decide network for custom chainConfig.
backend, err := ethBackend(db, &ipldeth.Config{ backend, err := NewEthBackend(db, &ipldEth.Config{
ChainConfig: chainCfg, ChainConfig: chainCfg,
GroupCacheConfig: &shared.GroupCacheConfig{ GroupCacheConfig: &ethServerShared.GroupCacheConfig{
StateDB: shared.GroupConfig{ StateDB: ethServerShared.GroupConfig{
Name: "cerc_validator", Name: "vulcanize_validator",
}, },
}, },
}) })
@ -222,44 +204,12 @@ func EthAPI(ctx context.Context, db *sqlx.DB, chainCfg *params.ChainConfig) (*ip
} }
backend.Config.ChainConfig = setChainConfig(genesisBlock.Hash()) backend.Config.ChainConfig = setChainConfig(genesisBlock.Hash())
} }
config := ipldeth.APIConfig{
SupportsStateDiff: false,
ForwardEthCalls: false,
ForwardGetStorageAt: false,
ProxyOnError: false,
StateDiffTimeout: 0,
}
return ipldeth.NewPublicEthAPI(backend, nil, config)
}
func ethBackend(db *sqlx.DB, c *ipldeth.Config) (*ipldeth.Backend, error) { return ipldEth.NewPublicEthAPI(backend, nil, false, false, false)
gcc := c.GroupCacheConfig
groupName := gcc.StateDB.Name
if groupName == "" {
groupName = ipldeth.StateDBGroupCacheName
}
r := ipldeth.NewRetriever(db)
ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{
Name: groupName,
Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024,
ExpiryDuration: time.Minute * time.Duration(gcc.StateDB.CacheExpiryInMins),
})
// Read only wrapper around ipfs-ethdb eth.Database implementation
ethDB = newDatabase(ethDB)
return &ipldeth.Backend{
DB: db,
Retriever: r,
EthDB: ethDB,
IpldTrieStateDatabase: ipldstate.NewDatabase(ethDB),
Config: c,
}, nil
} }
// fetchHeadBlockNumber gets the latest block number from the db // fetchHeadBlockNumber gets the latest block number from the db
func fetchHeadBlockNumber(ctx context.Context, api *ipldeth.PublicEthAPI) (uint64, error) { func fetchHeadBlockNumber(ctx context.Context, api *ipldEth.PublicEthAPI) (uint64, error) {
headBlock, err := api.B.BlockByNumber(ctx, rpc.LatestBlockNumber) headBlock, err := api.B.BlockByNumber(ctx, rpc.LatestBlockNumber)
if err != nil { if err != nil {
return 0, err return 0, err
@ -268,101 +218,58 @@ func fetchHeadBlockNumber(ctx context.Context, api *ipldeth.PublicEthAPI) (uint6
return headBlock.NumberU64(), nil return headBlock.NumberU64(), nil
} }
// writeStateDiffAt calls out to a statediffing geth client to fill in a gap in the index // applyTransaction attempts to apply a transaction to the given state database
func (s *Service) writeStateDiffAt(height uint64) error { // and uses the input parameters for its environment. It returns the stateDB of parent with applied transa
if !s.stateDiffMissingBlock { func applyTransaction(block *types.Block, backend *ipldEth.Backend) (*state.StateDB, error) {
return nil
}
var data json.RawMessage
params := statediff.Params{
IncludeBlock: true,
IncludeReceipts: true,
IncludeTD: true,
IncludeCode: true,
}
ctx, cancel := context.WithTimeout(context.Background(), s.stateDiffTimeout)
defer cancel()
log.Warnf("calling writeStateDiffAt at block %d", height)
if err := s.ethClient.CallContext(ctx, &data, "statediff_writeStateDiffAt", height, params); err != nil {
log.Errorf("writeStateDiffAt %d failed with err %s", height, err)
return err
}
return nil
}
// applyTransaction attempts to apply block transactions to the given state database
// and uses the input parameters for its environment. It returns the stateDB of parent with applied txs.
//
// Note: this skips the DAO hard fork refund
func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate.StateDB, error) {
if block.NumberU64() == 0 { if block.NumberU64() == 0 {
return nil, errors.New("no transaction in genesis") return nil, errors.New("no transaction in genesis")
} }
config := backend.Config.ChainConfig
// Create the parent state database // Create the parent state database
parentHash := block.ParentHash() parentHash := block.ParentHash()
nrOrHash := rpc.BlockNumberOrHash{BlockHash: &parentHash} parentRPCBlockHash := rpc.BlockNumberOrHash{
statedb, _, err := backend.IPLDTrieStateDBAndHeaderByNumberOrHash(context.Background(), nrOrHash) BlockHash: &parentHash,
RequireCanonical: false,
}
parent, _ := backend.BlockByNumberOrHash(context.Background(), parentRPCBlockHash)
if parent == nil {
return nil, fmt.Errorf("parent %#x not found", block.ParentHash())
}
stateDB, _, err := backend.StateAndHeaderByNumberOrHash(context.Background(), parentRPCBlockHash)
if err != nil { if err != nil {
return nil, fmt.Errorf("error accessing state DB: %w", err) return nil, err
} }
var gp core.GasPool signer := types.MakeSigner(backend.Config.ChainConfig, block.Number())
gp.AddGas(block.GasLimit())
blockContext := core.NewEVMBlockContext(block.Header(), backend, getAuthor(backend, block.Header())) for idx, tx := range block.Transactions() {
evm := vm.NewEVM(blockContext, vm.TxContext{}, statedb, config, vm.Config{}) // Assemble the transaction call message and return if the requested offset
signer := types.MakeSigner(config, block.Number(), block.Time()) msg, _ := tx.AsMessage(signer, block.BaseFee())
txContext := core.NewEVMTxContext(msg)
ctx := core.NewEVMBlockContext(block.Header(), backend, getAuthor(backend, block.Header()))
if beaconRoot := block.BeaconRoot(); beaconRoot != nil { // Not yet the searched for transaction, execute on top of the current state
ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) newEVM := vm.NewEVM(ctx, txContext, stateDB, backend.Config.ChainConfig, vm.Config{})
}
// Iterate over and process the individual transactions stateDB.Prepare(tx.Hash(), idx)
for i, tx := range block.Transactions() { if _, err := core.ApplyMessage(newEVM, msg, new(core.GasPool).AddGas(block.GasLimit())); err != nil {
msg, err := core.TransactionToMessage(tx, signer, block.BaseFee()) return nil, fmt.Errorf("transaction %#x failed: %w", tx.Hash(), err)
if err != nil {
return nil, fmt.Errorf("error converting transaction to message: %w", err)
}
statedb.SetTxContext(tx.Hash(), i)
// Create a new context to be used in the EVM environment.
evm.Reset(core.NewEVMTxContext(msg), statedb)
// Apply the transaction to the current state (included in the env).
if _, err := core.ApplyMessage(evm, msg, &gp); err != nil {
return nil, fmt.Errorf("error applying tx %#x: %w", tx.Hash(), err)
}
if config.IsByzantium(block.Number()) {
statedb.Finalise(true)
} else {
statedb.IntermediateRoot(config.IsEIP158(block.Number())).Bytes()
} }
} }
// Withdrawals processing. if backend.Config.ChainConfig.Ethash != nil {
for _, w := range block.Withdrawals() { accumulateRewards(backend.Config.ChainConfig, stateDB, block.Header(), block.Uncles())
// Convert amount from gwei to wei.
amount := new(uint256.Int).SetUint64(w.Amount)
amount = amount.Mul(amount, uint256.NewInt(params.GWei))
statedb.AddBalance(w.Address, amount)
} }
if config.Ethash != nil { return stateDB, nil
accumulateRewards(config, statedb, block.Header(), block.Uncles())
}
return statedb, nil
} }
// accumulateRewards credits the coinbase of the given block with the mining // accumulateRewards credits the coinbase of the given block with the mining
// reward. The total reward consists of the static block reward and rewards for // reward. The total reward consists of the static block reward and rewards for
// included uncles. The coinbase of each uncle block is also rewarded. // included uncles. The coinbase of each uncle block is also rewarded.
func accumulateRewards(config *params.ChainConfig, state *ipldstate.StateDB, header *types.Header, uncles []*types.Header) { func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) {
// Select the correct block reward based on chain progression // Select the correct block reward based on chain progression
blockReward := ethash.FrontierBlockReward blockReward := ethash.FrontierBlockReward
if config.IsByzantium(header.Number) { if config.IsByzantium(header.Number) {
@ -374,48 +281,32 @@ func accumulateRewards(config *params.ChainConfig, state *ipldstate.StateDB, hea
} }
// Accumulate the rewards for the miner and any included uncles // Accumulate the rewards for the miner and any included uncles
reward := new(big.Int).Set(blockReward.ToBig()) reward := new(big.Int).Set(blockReward)
r := new(big.Int) r := new(big.Int)
for _, uncle := range uncles { for _, uncle := range uncles {
r.Add(uncle.Number, big8) r.Add(uncle.Number, big8)
r.Sub(r, header.Number) r.Sub(r, header.Number)
r.Mul(r, blockReward.ToBig()) r.Mul(r, blockReward)
r.Div(r, big8) r.Div(r, big8)
state.AddBalance(uncle.Coinbase, uint256.MustFromBig(r)) state.AddBalance(uncle.Coinbase, r)
r.Div(blockReward.ToBig(), big32) r.Div(blockReward, big32)
reward.Add(reward, r) reward.Add(reward, r)
} }
state.AddBalance(header.Coinbase, uint256.MustFromBig(reward)) state.AddBalance(header.Coinbase, reward)
}
// ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root
// contract. This method is exported to be used in tests.
func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb *ipldstate.StateDB) {
// If EIP-4788 is enabled, we need to invoke the beaconroot storage contract with
// the new root
msg := &core.Message{
From: params.SystemAddress,
GasLimit: 30_000_000,
GasPrice: common.Big0,
GasFeeCap: common.Big0,
GasTipCap: common.Big0,
To: &params.BeaconRootsStorageAddress,
Data: beaconRoot[:],
}
vmenv.Reset(core.NewEVMTxContext(msg), statedb)
statedb.AddAddressToAccessList(params.BeaconRootsStorageAddress)
_, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560)
statedb.Finalise(true)
} }
func setChainConfig(ghash common.Hash) *params.ChainConfig { func setChainConfig(ghash common.Hash) *params.ChainConfig {
switch { switch {
case ghash == params.MainnetGenesisHash: case ghash == params.MainnetGenesisHash:
return params.MainnetChainConfig return params.MainnetChainConfig
case ghash == params.RopstenGenesisHash:
return params.RopstenChainConfig
case ghash == params.SepoliaGenesisHash: case ghash == params.SepoliaGenesisHash:
return params.SepoliaChainConfig return params.SepoliaChainConfig
case ghash == params.RinkebyGenesisHash:
return params.RinkebyChainConfig
case ghash == params.GoerliGenesisHash: case ghash == params.GoerliGenesisHash:
return params.GoerliChainConfig return params.GoerliChainConfig
default: default:
@ -423,7 +314,7 @@ func setChainConfig(ghash common.Hash) *params.ChainConfig {
} }
} }
func getAuthor(b *ipldeth.Backend, header *types.Header) *common.Address { func getAuthor(b *ipldEth.Backend, header *types.Header) *common.Address {
author, err := getEngine(b).Author(header) author, err := getEngine(b).Author(header)
if err != nil { if err != nil {
return nil return nil
@ -432,7 +323,7 @@ func getAuthor(b *ipldeth.Backend, header *types.Header) *common.Address {
return &author return &author
} }
func getEngine(b *ipldeth.Backend) consensus.Engine { func getEngine(b *ipldEth.Backend) consensus.Engine {
// TODO: add logic for other engines // TODO: add logic for other engines
if b.Config.ChainConfig.Clique != nil { if b.Config.ChainConfig.Clique != nil {
engine := clique.New(b.Config.ChainConfig.Clique, nil) engine := clique.New(b.Config.ChainConfig.Clique, nil)

13
pkg/validator/validator_suite_test.go Normal file
View File

@ -0,0 +1,13 @@
package validator_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestValidator(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Validator Suite")
}

134
pkg/validator/validator_test.go
View File

@ -1,134 +0,0 @@
package validator_test
import (
"context"
"math/big"
"testing"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
indexer_helpers "github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
helpers "github.com/cerc-io/plugeth-statediff/test_helpers"
sdtypes "github.com/cerc-io/plugeth-statediff/types"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
"github.com/jmoiron/sqlx"
// import server helpers for non-canonical chain data
server_mocks "github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
"github.com/cerc-io/ipld-eth-db-validator/v5/internal/chaingen"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/validator"
)
const (
chainLength = 10
startBlock = 1
)
var (
chainConfig = TestChainConfig
mockTD = big.NewInt(1337)
testDB = rawdb.NewMemoryDatabase()
)
func init() {
// The geth sync logs are noisy, silence them
log.SetDefault(log.NewLogger(log.DiscardHandler()))
}
func setupStateValidator(t *testing.T) *sqlx.DB {
// Make the test blockchain and state
gen := chaingen.DefaultGenContext(chainConfig, testDB)
blocks, receipts, chain := gen.MakeChain(chainLength)
t.Cleanup(func() {
chain.Stop()
})
indexer, err := helpers.NewIndexer(context.Background(), chainConfig, gen.Genesis.Hash(), TestDBConfig)
if err != nil {
t.Fatal(err)
}
// Insert some non-canonical data into the database so that we test our ability to discern canonicity
// Use a func here for defer
func() {
tx, err := indexer.PushBlock(server_mocks.MockBlock, server_mocks.MockReceipts,
server_mocks.MockBlock.Difficulty())
if err != nil {
t.Fatal(err)
}
defer tx.RollbackOnFailure(err)
err = tx.Submit()
if err != nil {
t.Fatal(err)
}
// The non-canonical header has a child
tx, err = indexer.PushBlock(server_mocks.MockChild, server_mocks.MockReceipts,
server_mocks.MockChild.Difficulty())
if err != nil {
t.Fatal(err)
}
defer tx.RollbackOnFailure(err)
ipld := sdtypes.IPLD{
CID: ipld.Keccak256ToCid(ipld.RawBinary, server_mocks.CodeHash.Bytes()).String(),
Content: server_mocks.ContractCode,
}
err = indexer.PushIPLD(tx, ipld)
if err != nil {
t.Fatal(err)
}
err = tx.Submit()
if err != nil {
t.Fatal(err)
}
}()
if err := helpers.IndexChain(indexer, helpers.IndexChainParams{
StateCache: chain.StateCache(),
Blocks: blocks,
Receipts: receipts,
TotalDifficulty: mockTD,
}); err != nil {
t.Fatal(err)
}
db := SetupDB()
t.Cleanup(func() {
if err := indexer_helpers.ClearSqlxDB(db); err != nil {
t.Fatal(err)
}
})
return db
}
func TestStateValidation(t *testing.T) {
db := setupStateValidator(t)
t.Run("Validator", func(t *testing.T) {
api, err := validator.EthAPI(context.Background(), db, chainConfig)
if err != nil {
t.Fatal(err)
}
for i := uint64(startBlock); i <= chainLength; i++ {
blockToBeValidated, err := api.B.BlockByNumber(context.Background(), rpc.BlockNumber(i))
if err != nil {
t.Fatal(err)
}
if blockToBeValidated == nil {
t.Fatal("block was not found")
}
err = validator.ValidateBlock(blockToBeValidated, api.B, i)
if err != nil {
t.Fatal(err)
}
}
})
}

12
scripts/get-block-number.sh
View File

@ -1,12 +0,0 @@
#!/bin/bash
set -eu
geth_endpoint="${1:-$ETH_HTTP_PATH}"
latest_block_hex=$(curl -s $geth_endpoint -X POST -H "Content-Type: application/json" \
--data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":42}' | \
python3 -c 'import json, sys; print(int(json.load(sys.stdin)["result"], 16))' \
)
printf "%d" $latest_block_hex

55
scripts/run-test-stack.sh
View File

@ -1,55 +0,0 @@
#!/bin/bash
set -ex -o pipefail
stack_dir=$(readlink -f "$1")
[[ -d "$stack_dir" ]]
CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
# By default assume we are running in the project root.
export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}"
laconic_so="laconic-so --verbose --stack $stack_dir"
# Don't run geth/plugeth in the debugger, it will swallow error backtraces
echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
# Passing this lets us run eth_call forwarding tests without running ipld-eth-db
echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env
if [[ -z $SKIP_BUILD ]]; then
# Prevent conflicting tty output
export BUILDKIT_PROGRESS=plain
$laconic_so setup-repositories \
--exclude git.vdb.to/cerc-io/ipld-eth-server
$laconic_so build-containers \
--exclude cerc/ipld-eth-server
fi
if ! $laconic_so deploy \
--exclude ipld-eth-server \
--env-file $CONFIG_DIR/stack.env \
--cluster test up
then
$laconic_so deploy --cluster test logs
exit 1
fi
set +x
# Get IPv4 endpoint of geth and bootnode file server
bootnode_endpoint=localhost:$(docker port test-fixturenet-eth-bootnode-geth-1 9898 | head -1 | cut -d':' -f2)
geth_endpoint=localhost:$(docker port test-fixturenet-eth-geth-1-1 8545 | head -1 | cut -d':' -f2)
# Extract the chain config and ID from genesis file
curl -s $bootnode_endpoint/geth.json | jq '.config' > "$CONFIG_DIR/chain.json"
# Output vars if we are running on Github
if [[ -n "$GITHUB_ENV" ]]; then
echo ETH_CHAIN_ID="$(jq '.chainId' $CONFIG_DIR/chain.json)" >> "$GITHUB_ENV"
echo ETH_CHAIN_CONFIG="$CONFIG_DIR/chain.json" >> "$GITHUB_ENV"
echo ETH_HTTP_PATH=$geth_endpoint >> "$GITHUB_ENV"
# Read a private key so we can send from a funded account
echo DEPLOYER_PRIVATE_KEY="$(curl -s $bootnode_endpoint/accounts.csv | head -1 | cut -d',' -f3)" >> "$GITHUB_ENV"
fi

15
scripts/run_integration_test.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash
set -e
set -o xtrace
export PGPASSWORD=password
export DATABASE_USER=vdbm
export DATABASE_PORT=8077
export DATABASE_PASSWORD=password
export DATABASE_HOSTNAME=127.0.0.1
export DATABASE_NAME=vulcanize_testing
# Wait for containers to be up and execute the integration test.
while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
make integrationtest

17
scripts/run_unit_test.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
set -e
# Clear up existing docker images and volume.
docker-compose down --remove-orphans --volumes
# Spin up TimescaleDB
docker-compose -f docker-compose.yml up -d migrations ipld-eth-db
sleep 45
# Run unit tests
go clean -testcache
PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test
# Clean up
docker-compose down --remove-orphans --volumes

91
test/README.md Normal file
View File

@ -0,0 +1,91 @@
# Test Instructions
## Setup
- For running integration tests:
- Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator), [go-ethereum](https://github.com/vulcanize/go-ethereum) and [ipld-eth-db](https://github.com/vulcanize/ipld-eth-db) repositories.
- Checkout [v4 release](https://github.com/vulcanize/ipld-eth-db/releases/tag/v4.2.0-alpha) in ipld-eth-db repo.
```bash
# In ipld-eth-db repo.
git checkout v4.2.0-alpha
```
- Checkout [v4 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.19-statediff-4.1.0-alpha) in go-ethereum repo.
```bash
# In go-ethereum repo.
git checkout v1.10.19-statediff-4.1.0-alpha
```
- Checkout working commit in stack-orchestrator repo.
```bash
# In stack-orchestrator repo.
git checkout f2fd766f5400fcb9eb47b50675d2e3b1f2753702
```
## Run
- Run unit tests:
```bash
# In ipld-eth-db-validator root directory.
./scripts/run_unit_test.sh
```
- Run integration tests:
- In stack-orchestrator repo:
- Create config file:
```bash
cd helper-scripts
./create-config.sh
```
A `config.sh` will be created in the root directory.
- Update/Edit the config file `config.sh`:
```bash
#!/bin/bash
# Path to ipld-eth-server repo.
vulcanize_ipld_eth_db=~/ipld-eth-db/
# Path to go-ethereum repo.
vulcanize_go_ethereum=~/go-ethereum
# Path to contract folder.
vulcanize_test_contract=~/ipld-eth-db-validator/test/contract
genesis_file_path='start-up-files/go-ethereum/genesis.json'
db_write=true
```
- Run stack-orchestrator:
```bash
# In stack-orchestrator root directory.
cd helper-scripts
./wrapper.sh \
-e docker \
-d ../docker/local/docker-compose-db-sharding.yml \
-d ../docker/local/docker-compose-go-ethereum.yml \
-d ../docker/local/docker-compose-contract.yml \
-v remove \
-p ../config.sh
```
- Run tests:
```bash
# In ipld-eth-db-validator root directory.
./scripts/run_integration_test.sh
```

19
test/compose-deployer.yml
View File

@ -1,19 +0,0 @@
# Runs the test contract deployment server
services:
contract-deployer:
restart: on-failure
image: cerc/ipld-eth-db-validator/test-contract-deployer:local
build: ./contract
networks:
- test_default
environment:
ETH_ADDR: "http://fixturenet-eth-geth-1:8545"
ETH_CHAIN_ID: $ETH_CHAIN_ID
DEPLOYER_PRIVATE_KEY: $DEPLOYER_PRIVATE_KEY
ports:
- 127.0.0.1:3000:3000
networks:
test_default:
external: true

8
test/contract/Dockerfile
View File

@ -1,5 +1,7 @@
# Downgrade from 18.16, see https://github.com/NomicFoundation/hardhat/issues/3877 FROM node:14
FROM node:20-slim
ARG ETH_ADDR
ENV ETH_ADDR $ETH_ADDR
WORKDIR /usr/src/app WORKDIR /usr/src/app
COPY package*.json ./ COPY package*.json ./
@ -9,4 +11,4 @@ RUN npm run compile && ls -lah
EXPOSE 3000 EXPOSE 3000
ENTRYPOINT ["node", "src/index.js"] ENTRYPOINT ["npm", "start"]

10
test/contract/contracts/GLDToken.sol
View File

@ -1,16 +1,10 @@
pragma solidity ^0.8.25; pragma solidity ^0.8.0;
import "@openzeppelin/contracts/token/ERC20/ERC20.sol"; import "@openzeppelin/contracts/token/ERC20/ERC20.sol";
contract GLDToken is ERC20 { contract GLDToken is ERC20 {
constructor() ERC20("Gold", "GLD") { constructor() ERC20("Gold", "GLD") {
_mint(msg.sender, 1000000000000000000000); _mint(msg.sender, 1000000000000000000000);
} }
function destroy() public { function destroy() public {
(bool ok, ) = payable(msg.sender).call{value: address(this).balance}(""); selfdestruct(payable(msg.sender));
require(ok, "ETH transfer failed");
_burn(msg.sender, balanceOf(msg.sender));
} }
} }

38
test/contract/contracts/Test.sol
View File

@ -2,28 +2,28 @@
pragma solidity ^0.8.0; pragma solidity ^0.8.0;
contract Test { contract Test {
event logPut (address, uint256); address payable owner;
address payable owner; modifier onlyOwner {
mapping(address => uint256) public data; require(
msg.sender == owner,
"Only owner can call this function."
);
_;
}
modifier onlyOwner { uint256[100] data;
require(msg.sender == owner, "Only owner can call this function.");
_;
}
constructor() { constructor() {
owner = payable(msg.sender); owner = payable(msg.sender);
} data = [1];
}
function Put(uint256 value) public { function Put(uint256 addr, uint256 value) public {
emit logPut(msg.sender, value); data[addr] = value;
}
data[msg.sender] = value; function close() public onlyOwner {
} selfdestruct(owner);
}
function close() public onlyOwner {
(bool ok, ) = owner.call{value: address(this).balance}("");
require(ok, "ETH transfer failed");
}
} }

36
test/contract/hardhat.config.js
View File

@ -16,31 +16,17 @@ task("accounts", "Prints the list of accounts", async () => {
/** /**
* @type import('hardhat/config').HardhatUserConfig * @type import('hardhat/config').HardhatUserConfig
*/ */
const localNetwork = {
url: process.env.ETH_ADDR || "http://127.0.0.1:8545",
chainId: Number(process.env.ETH_CHAIN_ID) || 99,
};
if (process.env.DEPLOYER_PRIVATE_KEY) {
localNetwork["accounts"] = [process.env.DEPLOYER_PRIVATE_KEY];
}
module.exports = { module.exports = {
solidity: { solidity: "0.8.0",
version: "0.8.25",
settings: {
outputSelection: {
'*': {
'*': [
'abi', 'storageLayout',
]
}
}
}
},
networks: { networks: {
local: localNetwork local: {
}, url: 'http://127.0.0.1:8545',
defaultNetwork: "local" chainId: 99
},
docker: {
url: process.env.ETH_ADDR,
chainId: 99
}
}
}; };

17649
test/contract/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

17
test/contract/package.json
View File

@ -4,21 +4,22 @@
"main": "index.js", "main": "index.js",
"scripts": { "scripts": {
"compile": "npx hardhat compile", "compile": "npx hardhat compile",
"start": "node src/index.js" "start": "HARDHAT_NETWORK=docker node src/index.js",
"start:local": "ETH_ADDR=http://127.0.0.1:8545 npm run start",
"test": "echo \"Error: no test specified\" && exit 1"
}, },
"keywords": [], "keywords": [],
"author": "", "author": "",
"license": "ISC", "license": "ISC",
"description": "Solidity contract deployment server for integration testing", "description": "",
"dependencies": { "dependencies": {
"@openzeppelin/contracts": "^5.0.2", "@openzeppelin/contracts": "^4.0.0",
"fastify": "^4.26.2", "fastify": "^3.14.2",
"hardhat": "^2.22.3", "hardhat": "^2.2.0"
"solidity-create2-deployer": "^0.4.0"
}, },
"devDependencies": { "devDependencies": {
"@nomiclabs/hardhat-ethers": "^2.2.3", "@nomiclabs/hardhat-ethers": "^2.0.2",
"@nomiclabs/hardhat-waffle": "^2.0.4", "@nomiclabs/hardhat-waffle": "^2.0.1",
"chai": "^4.3.4", "chai": "^4.3.4",
"ethereum-waffle": "^3.3.0", "ethereum-waffle": "^3.3.0",
"ethers": "^5.1.0" "ethers": "^5.1.0"

18
test/contract/scripts/deploy.js Normal file
View File

@ -0,0 +1,18 @@
const hre = require("hardhat");
async function main() {
// await hre.run('compile');
// We get the contract to deploy
const GLDToken = await hre.ethers.getContractFactory("GLDToken");
const token = await GLDToken.deploy();
await token.deployed();
console.log("GLDToken deployed to:", token.address, token.deployTransaction.hash);
}
// We recommend this pattern to be able to use async/await everywhere
// and properly handle errors.
main()
.then(() => process.exit(0))
.catch(error => {
console.error(error);
process.exit(1);
});

36
test/contract/scripts/sample-script.js Normal file
View File

@ -0,0 +1,36 @@
// We require the Hardhat Runtime Environment explicitly here. This is optional
// but useful for running the script in a standalone fashion through `node <script>`.
//
// When running the script with `hardhat run <script>` you'll find the Hardhat
// Runtime Environment's members available in the global scope.
const hre = require("hardhat");
async function main() {
// Hardhat always runs the compile task when running scripts with its command
// line interface.
//
// If this script is run directly using `node` you may want to call compile
// manually to make sure everything is compiled
// await hre.run('compile');
// We get the contract to deploy
const Greeter = await hre.ethers.getContractFactory("Greeter");
const greeter = await Greeter.deploy("Hello, Hardhat!");
await greeter.deployed();
console.log("Greeter deployed to:", greeter.address, "; tx hash: ", greeter.deployTransaction.hash);
const result = await greeter.setGreeting("Hello 123!");
console.log("Greeter updated", "; tx hash: ", result.hash, "; block hash: ", result.blockHash);
}
// We recommend this pattern to be able to use async/await everywhere
// and properly handle errors.
main()
.then(() => process.exit(0))
.catch(error => {
console.error(error);
process.exit(1);
});

153
test/contract/src/index.js
View File

@ -1,87 +1,118 @@
const fastify = require('fastify')({ logger: true }); const fastify = require('fastify')({ logger: true });
const hre = require("hardhat"); const hre = require("hardhat");
// readiness check // readiness check
fastify.get('/v1/healthz', async (req, reply) => { fastify.get('/v1/healthz', async (req, reply) => {
reply reply
.code(200) .code(200)
.header('Content-Type', 'application/json; charset=utf-8') .header('Content-Type', 'application/json; charset=utf-8')
.send({ success: true }) .send({ success: true })
}); });
fastify.get('/v1/deployContract', async (req, reply) => {
const GLDToken = await hre.ethers.getContractFactory("GLDToken");
const token = await GLDToken.deploy();
await token.deployed();
return {
address: token.address,
txHash: token.deployTransaction.hash,
blockNumber: token.deployTransaction.blockNumber,
blockHash: token.deployTransaction.blockHash,
}
});
fastify.get('/v1/destroyContract', async (req, reply) => {
const addr = req.query.addr;
const Token = await hre.ethers.getContractFactory("GLDToken");
const token = await Token.attach(addr);
await token.destroy();
const blockNum = await hre.ethers.provider.getBlockNumber()
return {
blockNumber: blockNum,
}
})
fastify.get('/v1/sendEth', async (req, reply) => { fastify.get('/v1/sendEth', async (req, reply) => {
const to = req.query.to; const to = req.query.to;
const value = hre.ethers.utils.parseEther(req.query.value); const value = req.query.value;
const owner = await hre.ethers.getSigner(); const [owner] = await hre.ethers.getSigners();
const tx = await owner.sendTransaction({to, value}).then(tx => tx.wait()); const tx = await owner.sendTransaction({
to,
value: hre.ethers.utils.parseEther(value)
});
await tx.wait(1)
return { // console.log(tx);
from: tx.from, // const coinbaseBalance = await hre.ethers.provider.getBalance(owner.address);
to: tx.to, // const receiverBalance = await hre.ethers.provider.getBalance(to);
txHash: tx.hash, // console.log(coinbaseBalance.toString(), receiverBalance.toString());
blockNumber: tx.blockNumber,
blockHash: tx.blockHash, return {
} from: tx.from,
to: tx.to,
//value: tx.value.toString(),
txHash: tx.hash,
blockNumber: tx.blockNumber,
blockHash: tx.blockHash,
}
}); });
function contractCreator(name) { fastify.get('/v1/deployTestContract', async (req, reply) => {
return async (req, reply) => { const testContract = await hre.ethers.getContractFactory("Test");
const contract = await hre.ethers.getContractFactory(name); const test = await testContract.deploy();
const instance = await contract.deploy(); await test.deployed();
const rct = await instance.deployTransaction.wait();
return { return {
address: instance.address, address: test.address,
txHash: rct.transactionHash, txHash: test.deployTransaction.hash,
blockNumber: rct.blockNumber, blockNumber: test.deployTransaction.blockNumber,
blockHash: rct.blockHash, blockHash: test.deployTransaction.blockHash,
} }
} });
}
function contractDestroyer(name) {
return async (req, reply) => {
const addr = req.query.addr;
const contract = await hre.ethers.getContractFactory(name);
const instance = contract.attach(addr);
const rct = await instance.destroy().then(tx => tx.wait());
return {
blockNumber: rct.blockNumber,
}
}
}
fastify.get('/v1/deployContract', contractCreator("GLDToken"));
fastify.get('/v1/destroyContract', contractDestroyer("GLDToken"));
fastify.get('/v1/deployTestContract', contractCreator("Test"));
fastify.get('/v1/destroyTestContract', contractDestroyer("Test"));
fastify.get('/v1/putTestValue', async (req, reply) => { fastify.get('/v1/putTestValue', async (req, reply) => {
const addr = req.query.addr; const addr = req.query.addr;
const value = req.query.value; const index = req.query.index;
const value = req.query.value;
const testContract = await hre.ethers.getContractFactory("Test"); const testContract = await hre.ethers.getContractFactory("Test");
const test = await testContract.attach(addr); const test = await testContract.attach(addr);
const rct = await test.Put(value).then(tx => tx.wait()); const tx = await test.Put(index, value);
const receipt = await tx.wait();
return { return {
blockNumber: rct.blockNumber, blockNumber: receipt.blockNumber,
} }
}); });
fastify.get('/v1/destroyTestContract', async (req, reply) => {
const addr = req.query.addr;
const testContract = await hre.ethers.getContractFactory("Test");
const test = await testContract.attach(addr);
await test.destroy();
const blockNum = await hre.ethers.provider.getBlockNumber()
return {
blockNumber: blockNum,
}
})
async function main() { async function main() {
try { try {
await fastify.listen({ port: 3000, host: '0.0.0.0' }); await fastify.listen(3000, '0.0.0.0');
} catch (err) { } catch (err) {
fastify.log.error(err); fastify.log.error(err);
process.exit(1); process.exit(1);
} }
} }
process.on('SIGINT', () => fastify.close().then(() => process.exit(1)));
main(); main();

13
test/contract/test/basic-test.js
View File

@ -1,13 +0,0 @@
import { expect } from 'chai';
describe("GLDToken", function() {
it("Should return the owner's balance", async function() {
const Token = await ethers.getContractFactory("GLDToken");
const token = await Token.deploy();
await token.deployed();
const [owner] = await ethers.getSigners();
const balance = await token.balanceOf(owner.address);
expect(balance).to.equal('1000000000000000000000');
expect(await token.totalSupply()).to.equal(balance); });
});

14
test/contract/test/sample-test.js Normal file
View File

@ -0,0 +1,14 @@
const { expect } = require("chai");
describe("Greeter", function() {
it("Should return the new greeting once it's changed", async function() {
const Greeter = await ethers.getContractFactory("Greeter");
const greeter = await Greeter.deploy("Hello, world!");
await greeter.deployed();
expect(await greeter.greet()).to.equal("Hello, world!");
await greeter.setGreeting("Hola, mundo!");
expect(await greeter.greet()).to.equal("Hola, mundo!");
});
});

54
test/helper.go Normal file
View File

@ -0,0 +1,54 @@
package integration
import (
"encoding/json"
"fmt"
"net/http"
ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
)
type PutResult struct {
BlockNumber int64 `json:"blockNumber"`
}
const srvUrl = "http://localhost:3000"
func DeployTestContract() (*ethServerIntegration.ContractDeployed, error) {
ethServerIntegration.DeployContract()
res, err := http.Get(fmt.Sprintf("%s/v1/deployTestContract", srvUrl))
if err != nil {
return nil, err
}
defer res.Body.Close()
var contract ethServerIntegration.ContractDeployed
decoder := json.NewDecoder(res.Body)
return &contract, decoder.Decode(&contract)
}
func PutTestValue(addr string, index, value int) (*PutResult, error) {
res, err := http.Get(fmt.Sprintf("%s/v1/putTestValue?addr=%s&index=%d&value=%d", srvUrl, addr, index, value))
if err != nil {
return nil, err
}
var blockNumber PutResult
decoder := json.NewDecoder(res.Body)
return &blockNumber, decoder.Decode(&blockNumber)
}
func DestroyTestContract(addr string) (*ethServerIntegration.ContractDestroyed, error) {
res, err := http.Get(fmt.Sprintf("%s/v1/destroyTestContract?addr=%s", srvUrl, addr))
if err != nil {
return nil, err
}
defer res.Body.Close()
var data ethServerIntegration.ContractDestroyed
decoder := json.NewDecoder(res.Body)
return &data, decoder.Decode(&data)
}

19
test/integration_suite_test.go Normal file
View File

@ -0,0 +1,19 @@
package integration_test
import (
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestIntegration(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "integration test suite")
}
var _ = BeforeSuite(func() {
logrus.SetOutput(ioutil.Discard)
})

108
test/integration_test.go Normal file
View File

@ -0,0 +1,108 @@
package integration_test
import (
"context"
"sync"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
integration "github.com/vulcanize/ipld-eth-db-validator/test"
"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
)
const (
blockNum = 1
trail = 0
validatorSleepInterval = uint(5)
)
var (
testAddresses = []string{
"0x1111111111111111111111111111111111111112",
"0x1ca7c995f8eF0A2989BbcE08D5B7Efe50A584aa1",
"0x9a4b666af23a2cdb4e5538e1d222a445aeb82134",
"0xF7C7AEaECD2349b129d5d15790241c32eeE4607B",
"0x992b6E9BFCA1F7b0797Cee10b0170E536EAd3532",
"0x29ed93a7454Bc17a8D4A24D0627009eE0849B990",
"0x66E3dCA826b04B5d4988F7a37c91c9b1041e579D",
"0x96288939Ac7048c27E0E087b02bDaad3cd61b37b",
"0xD354280BCd771541c935b15bc04342c26086FE9B",
"0x7f887e25688c274E77b8DeB3286A55129B55AF14",
}
)
var _ = Describe("Integration test", func() {
ctx := context.Background()
var contract *ethServerIntegration.ContractDeployed
var err error
sleepInterval := 2 * time.Second
timeout := 4 * time.Second
db := shared.SetupDB()
cfg := validator.Config{
DB: db,
BlockNum: blockNum,
Trail: trail,
SleepInterval: validatorSleepInterval,
ChainCfg: validator.IntegrationTestChainConfig,
}
validationProgressChan := make(chan uint64)
service := validator.NewService(&cfg, validationProgressChan)
wg := new(sync.WaitGroup)
It("test init", func() {
wg.Add(1)
go service.Start(ctx, wg)
// Deploy a dummy contract as the first contract might get deployed at block number 0
_, _ = ethServerIntegration.DeployContract()
time.Sleep(sleepInterval)
})
defer It("test teardown", func() {
service.Stop()
wg.Wait()
Expect(validationProgressChan).To(BeClosed())
})
Describe("Validate state", func() {
It("performs validation on contract deployment", func() {
contract, err = integration.DeployTestContract()
Expect(err).ToNot(HaveOccurred())
time.Sleep(sleepInterval)
Expect(validationProgressChan).ToNot(BeClosed())
Eventually(validationProgressChan, timeout).Should(Receive(Equal(uint64(contract.BlockNumber))))
})
It("performs validation on contract transactions", func() {
for i := 0; i < 10; i++ {
res, txErr := integration.PutTestValue(contract.Address, i, i)
Expect(txErr).ToNot(HaveOccurred())
time.Sleep(sleepInterval)
Expect(validationProgressChan).ToNot(BeClosed())
Eventually(validationProgressChan, timeout).Should(Receive(Equal(uint64(res.BlockNumber))))
}
})
It("performs validation on eth transfer transactions", func() {
for _, address := range testAddresses {
tx, txErr := ethServerIntegration.SendEth(address, "0.01")
Expect(txErr).ToNot(HaveOccurred())
time.Sleep(sleepInterval)
Expect(validationProgressChan).ToNot(BeClosed())
Eventually(validationProgressChan, timeout).Should(Receive(Equal(uint64(tx.BlockNumber))))
}
})
})
})

117
validator_test/chain_maker.go Normal file
View File

@ -0,0 +1,117 @@
package validator_test
import (
"crypto/ecdsa"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
)
// Test variables
var (
Testdb = rawdb.NewMemoryDatabase()
TestBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
TestBankFunds = big.NewInt(100000000)
Genesis = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
DeploymentTxData = common.Hex2Bytes("0x608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
ContractCode = common.Hex2Bytes("0x608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
CodeHash = crypto.Keccak256Hash(ContractCode)
ContractAddr common.Address
IndexZero = "0000000000000000000000000000000000000000000000000000000000000000"
IndexOne = "0000000000000000000000000000000000000000000000000000000000000001"
ContractSlotPosition = common.FromHex(IndexOne)
ContractSlotKeyHash = crypto.Keccak256Hash(ContractSlotPosition)
MiningReward = big.NewInt(2000000000000000000)
)
/* test function signatures
put function sig: 65f3c31a
close function sig: 43d726d6
data function sig: 73d4a13a
*/
func createLondonTransaction(block *core.BlockGen, addr *common.Address, key *ecdsa.PrivateKey) *types.Transaction {
londonTrx := types.NewTx(&types.DynamicFeeTx{
ChainID: validator.TestChainConfig.ChainID,
Nonce: block.TxNonce(*addr),
GasTipCap: big.NewInt(50),
GasFeeCap: big.NewInt(1000000000),
Gas: 21000,
To: addr,
Value: big.NewInt(1000),
Data: []byte{},
})
transactionSigner := types.MakeSigner(validator.TestChainConfig, block.Number())
signedTrx1, err := types.SignTx(londonTrx, transactionSigner, key)
if err != nil {
log.Fatal(err.Error())
}
return signedTrx1
}
// MakeChain creates a chain of n blocks starting at and including parent.
// the returned hash chain is ordered head->parent.
func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, []types.Receipts, *core.BlockChain) {
config := validator.TestChainConfig
blocks, receipts := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
chain, _ := core.NewBlockChain(Testdb, nil, validator.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
return append([]*types.Block{parent}, blocks...), receipts, chain
}
func TestChainGen(i int, block *core.BlockGen) {
signer := types.HomesteadSigner{}
switch i {
case 0:
// In block 1, the test bank sends account #1 some ether.
tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey)
block.AddTx(tx)
case 1:
// In block 2, the test bank sends some more ether to account #1.
// Account1Addr passes it on to account #2.
// Account1Addr creates a test contract.
tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey)
nonce := block.TxNonce(Account1Addr)
tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
nonce++
tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), DeploymentTxData), signer, Account1Key)
ContractAddr = crypto.CreateAddress(Account1Addr, nonce)
block.AddTx(tx1)
block.AddTx(tx2)
block.AddTx(tx3)
case 2:
block.SetCoinbase(Account2Addr)
data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000003")
tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
block.AddTx(tx)
case 3:
block.SetCoinbase(Account2Addr)
data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000009")
tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
block.AddTx(tx)
case 4:
block.SetCoinbase(Account1Addr)
data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000000")
tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
block.AddTx(tx)
case 5:
block.AddTx(createLondonTransaction(block, &Account1Addr, Account1Key))
case 6:
block.AddTx(createLondonTransaction(block, &Account2Addr, Account2Key))
}
}

19
validator_test/validator_suite_test.go Normal file
View File

@ -0,0 +1,19 @@
package validator_test_test
import (
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestETHSuite(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "eth ipld validator eth suite test")
}
var _ = BeforeSuite(func() {
logrus.SetOutput(ioutil.Discard)
})

129
validator_test/validator_test.go Normal file
View File

@ -0,0 +1,129 @@
package validator_test_test
import (
"context"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/statediff"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/jmoiron/sqlx"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
"github.com/vulcanize/ipld-eth-db-validator/validator_test"
)
const (
chainLength = 20
blockHeight = 1
trail = 2
)
var _ = Describe("eth state reading tests", func() {
var (
blocks []*types.Block
receipts []types.Receipts
chain *core.BlockChain
db *sqlx.DB
chainConfig = validator.TestChainConfig
mockTD = big.NewInt(1337)
)
It("test init", func() {
db = shared.SetupDB()
transformer := shared.SetupTestStateDiffIndexer(context.Background(), chainConfig, validator_test.Genesis.Hash())
// make the test blockchain (and state)
blocks, receipts, chain = validator_test.MakeChain(chainLength, validator_test.Genesis, validator_test.TestChainGen)
params := statediff.Params{
IntermediateStateNodes: true,
IntermediateStorageNodes: true,
}
// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
builder := statediff.NewBuilder(chain.StateCache())
for i, block := range blocks {
var args statediff.Args
var rcts types.Receipts
if i == 0 {
args = statediff.Args{
OldStateRoot: common.Hash{},
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
} else {
args = statediff.Args{
OldStateRoot: blocks[i-1].Root(),
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
rcts = receipts[i-1]
}
diff, err := builder.BuildStateDiffObject(args, params)
Expect(err).ToNot(HaveOccurred())
tx, err := transformer.PushBlock(block, rcts, mockTD)
Expect(err).ToNot(HaveOccurred())
for _, node := range diff.Nodes {
err := transformer.PushStateNode(tx, node, block.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
}
// Insert some non-canonical data into the database so that we test our ability to discern canonicity
indexAndPublisher := shared.SetupTestStateDiffIndexer(context.Background(), chainConfig, validator_test.Genesis.Hash())
tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
// The non-canonical header has a child
tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
Expect(err).ToNot(HaveOccurred())
hash := sdtypes.CodeAndCodeHash{
Hash: test_helpers.CodeHash,
Code: test_helpers.ContractCode,
}
err = indexAndPublisher.PushCodeAndCodeHash(tx, hash)
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
defer It("test teardown", func() {
shared.TearDownDB(db)
chain.Stop()
})
Describe("state_validation", func() {
It("Validator", func() {
api, err := validator.EthAPI(context.Background(), db, validator.TestChainConfig)
Expect(err).ToNot(HaveOccurred())
for i := uint64(blockHeight); i <= chainLength-trail; i++ {
err = validator.ValidateBlock(context.Background(), api, i)
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateReferentialIntegrity(db, i)
Expect(err).ToNot(HaveOccurred())
}
})
})
})