Compare commits

..

No commits in common. "v5" and "sharding" have entirely different histories.
v5 ... sharding

58 changed files with 17907 additions and 5817 deletions

6
.github/workflows/on-pr.yml vendored Normal file
View File

@ -0,0 +1,6 @@
name: Run all tests
on: [pull_request]
jobs:
run-tests:
uses: ./.github/workflows/tests.yml

View File

@ -1,90 +0,0 @@
name: Unit and integration tests
on:
# workflow_call:
# Job headers are hidden when not top-level - run them directly for readability until fixed:
# https://github.com/go-gitea/gitea/issues/26736
pull_request:
branches: '*'
push:
branches:
- main
- ci-test
env:
SO_VERSION: v1.1.0-36d4969-202407091537
FIXTURENET_ETH_STACKS_REF: main
jobs:
unit-tests:
name: Run unit tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version-file: 'go.mod'
check-latest: true
- name: Run DB container
run: |
docker compose -f test/compose-db.yml up --wait --quiet-pull
- name: Run tests
run: go test -p 1 -v ./pkg/...
integration-tests:
name: Run integration tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version-file: 'go.mod'
check-latest: true
# At present the stock setup-python action fails on Linux/aarch64
# Conditional steps below workaroud this by using deadsnakes for that case only
- name: "Install Python for ARM on Linux"
if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
uses: deadsnakes/action@v3.0.1
with:
python-version: 3.11
- name: "Install Python cases other than ARM on Linux"
if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install stack-orchestrator
uses: actions/checkout@v3
with:
repository: cerc-io/stack-orchestrator
ref: ${{ env.SO_VERSION }}
path: ./stack-orchestrator
- run: pip install ./stack-orchestrator
- name: Clone fixturenet stack repo
uses: actions/checkout@v4
with:
repository: cerc-io/fixturenet-eth-stacks
ref: ${{ env.FIXTURENET_ETH_STACKS_REF }}
path: ./fixturenet-eth-stacks
progress: false
- name: Run testnet stack
env:
CERC_GO_AUTH_TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth
- name: Run contract deployer
run: |
docker compose -f test/compose-deployer.yml up --wait --quiet-pull
- name: Wait for testnet
run: |
# Start validator at current head, but not before Merge (block 1 on test chain)
while
echo "Waiting for chain head to progress..."
height=$(./scripts/get-block-number.sh $ETH_HTTP_PATH)
[[ "$height" -lt 2 ]];
do sleep 5; done
echo "Chain has reached block $height"
echo VALIDATE_FROM_BLOCK=$height >> "$GITHUB_ENV"
- name: Run tests
run: |
go test -v -p 1 ./integration/... -timeout=20m

91
.github/workflows/tests.yml vendored Normal file
View File

@ -0,0 +1,91 @@
name: Test the stack.
on:
workflow_call:
env:
STACK_ORCHESTRATOR_REF: "f2fd766f5400fcb9eb47b50675d2e3b1f2753702"
GO_ETHEREUM_REF: "v1.10.19-statediff-4.1.0-alpha" # Use the tag, we are going to download the bin not build it.
IPLD_ETH_DB_REF: "b59505eab252670c622b42ce60621e9747fb64f9"
jobs:
integrationtest:
name: Run integration tests
env:
GOPATH: /tmp/go
DB_WRITE: true
runs-on: ubuntu-latest
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- uses: actions/setup-go@v3
with:
go-version: ">=1.18.0"
check-latest: true
- uses: actions/checkout@v2
with:
path: "./ipld-eth-db-validator"
- uses: actions/checkout@v2
with:
ref: ${{ env.STACK_ORCHESTRATOR_REF }}
path: "./stack-orchestrator/"
repository: vulcanize/stack-orchestrator
- uses: actions/checkout@v2
with:
ref: ${{ env.IPLD_ETH_DB_REF }}
repository: vulcanize/ipld-eth-db
path: "./ipld-eth-db/"
- name: Create config file
run: |
echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-db-validator/test/contract >> ./config.sh
echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> ./config.sh
echo db_write=$DB_WRITE >> ./config.sh
cat ./config.sh
- name: Download Geth
run: |
cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
wget https://github.com/vulcanize/go-ethereum/releases/download/${{env.GO_ETHEREUM_REF}}/geth-linux-amd64
- name: Run docker compose
run: |
docker-compose \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
-f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
--env-file "$GITHUB_WORKSPACE/config.sh" \
up -d --build
- name: Run integration test.
run: |
cd $GITHUB_WORKSPACE/ipld-eth-db-validator
./scripts/run_integration_test.sh
unittest:
name: Run unit tests
runs-on: ubuntu-latest
steps:
- name: Create GOPATH
run: mkdir -p /tmp/go
- uses: actions/setup-go@v3
with:
go-version: ">=1.18.0"
check-latest: true
- name: Checkout code
uses: actions/checkout@v2
- name: Spin up database
run: |
docker-compose up -d
- name: Run unit tests
run: |
sleep 30
PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test

1
.gitignore vendored
View File

@ -1,3 +1,2 @@
ipld-eth-db-validator
.vscode
.idea

View File

@ -1,14 +1,21 @@
CONTRACTS_DIR := ./test/contract/contracts
CONTRACTS_OUTPUT_DIR := ./internal/testdata/build
BIN = $(GOPATH)/bin
BASE = $(GOPATH)/src/$(PACKAGE)
PKGS = go list ./... | grep -v "^vendor/"
CONTRACTS := GLDToken Test
# Tools
contracts: $(foreach C,$(CONTRACTS), $(CONTRACTS_OUTPUT_DIR)/$C.bin $(CONTRACTS_OUTPUT_DIR)/$C.abi)
.PHONY: contracts
.PHONY: integrationtest
integrationtest: | $(GOOSE)
go vet ./...
go fmt ./...
go run github.com/onsi/ginkgo/ginkgo -r test/ -v
test: contracts
go test -p 1 -v ./pkg/...
.PHONY: test
test: | $(GOOSE)
go vet ./...
go fmt ./...
go run github.com/onsi/ginkgo/ginkgo -r validator_test/ -v
clean:
rm $(CONTRACTS_OUTPUT_DIR)/*.bin $(CONTRACTS_OUTPUT_DIR)/*.abi
build:
go fmt ./...
GO111MODULE=on go build

151
README.md
View File

@ -1,124 +1,75 @@
# ipld-eth-db-validator
- [Validator-README](#validator-readme)
- [Overview](#overview)
- [Intention for the Validator](#intention-for-the-validator)
- [Edge Cases](#edge-cases)
- [Instructions for Testing](#instructions-for-testing)
- [Code Overview](#code-overview)
- [Known Bugs](#known-bugs)
- [Tests on 03/03/22](#tests-on-03-03-22)
- [Set Up](#set-up)
- [Testing Failures](#testing-failures)
> `ipld-eth-db-validator` performs validation checks on indexed Ethereum IPLD objects in a Postgres database:
> * Attempt to apply transactions in each block and validate resultant block hash
> * Check referential integrity between IPLD blocks and index tables
<small><i><a href='http://ecotrust-canada.github.io/markdown-toc/'>Table of contents generated with markdown-toc</a></i></small>
## Setup
# Overview
Build the binary:
This repository contains the validator. The purpose of the validator is to ensure that the data in the Core Postgres database match the data on the blockchain.
```bash
make build
```
# Intention for the Validator
## Configuration
The perfect scenario for the validator is as follows:
An example config file:
1. The validator will have the capacity to perform historical checks for the Core Postgres database. Users can contain these historical checks to specified configurations (block range).
2. The validator will validate a certain number of trailing blocks, `t`, trailing the head, `n`. Therefore the validator will constantly perform real-time validation starting at `n` and ending at `n - t`.
3. The validator validates the IPLD blocks in the Core Database; it will update the core database to indicate that the validator validated it.
```toml
[database]
# db credentials
name = "cerc_public" # DATABASE_NAME
hostname = "localhost" # DATABASE_HOSTNAME
port = 5432 # DATABASE_PORT
user = "vdbm" # DATABASE_USER
password = "..." # DATABASE_PASSWORD
## Edge Cases
[validate]
# block height to initiate database validation at
fromBlock = 1 # VALIDATE_FROM_BLOCK (default: 1)
# number of blocks to trail behind the head
trail = 64 # VALIDATE_TRAIL (default: 64)
# retry interval after validator has caught up to (head-trail) height (in sec)
retryInterval = 10 # VALIDATE_RETRY_INTERVAL (default: 10)
We must consider the following edge cases for the validator.
# whether to perform a statediffing call on a missing block
stateDiffMissingBlock = true # (default: false)
# statediffing call timeout period (in sec)
stateDiffTimeout = 240 # (default: 240)
- There are three different data types that the validator must account for.
[ethereum]
# node info
# path to json chain config (optional)
chainConfig = "" # ETH_CHAIN_CONFIG
# eth chain id for config (overridden by chainConfig)
chainID = "1" # ETH_CHAIN_ID (default: 1)
# http RPC endpoint URL for a statediffing node
httpPath = "localhost:8545" # ETH_HTTP_PATH
# Instructions for Testing
[prom]
# prometheus metrics
metrics = true # PROM_METRICS (default: false)
http = true # PROM_HTTP (default: false)
httpAddr = "0.0.0.0" # PROM_HTTP_ADDR (default: 127.0.0.1)
httpPort = "9001" # PROM_HTTP_PORT (default: 9001)
dbStats = true # PROM_DB_STATS (default: false)
Follow steps in [test/README.md](./test/README.md)
[log]
# log level (trace, debug, info, warn, error, fatal, panic)
level = "info" # LOG_LEVEL (default: info)
# file path for logging, leave unset to log to stdout
file = "" # LOG_FILE_PATH
```
# Code Overview
This section will provide some insight into specific files and their purpose.
* The validation process trails behind the latest block number in the database by config parameter `validate.trail`.
- `validator_test/chain_maker.go` - This file contains the code for creating a “test” blockchain.
- `validator_test/validator_test.go` - This file contains testing to validate the validator. It leverages `chain_maker.go` to create a blockchain to validate.
- `pkg/validator/validator.go` - This file contains most of the core logic for the validator.
* If the validator has caught up to (head-trail) height, it waits for a configured time interval (`validate.retryInterval`) before again querying the database.
# Known Bugs
* If the validator encounters a missing block (gap) in the database, it makes a `writeStateDiffAt` call to the configured statediffing endpoint (`ethereum.httpPath`) if `validate.stateDiffMissingBlock` is set to `true`. Here it is assumed that the statediffing node pointed to is writing out to the database.
1. The validator is improperly handling missing headers from the database.
1. Scenario
1. The IPLD blocks from the mock blockchain are inserted into the Postgres Data.
2. The validator runs, and all tests pass.
3. Users manually remove the last few rows from the database.
4. The validator runs, and all tests pass - This behavior is neither expected nor wanted.
### Local Setup
# Tests on 03/03/22
* Create a chain config file `chain.json` according to chain config in genesis json file used by local geth.
The tests highlighted below were conducted to validate the initial behavior of the validator.
Example:
## Set Up
```json
{
"chainId": 41337,
"homesteadBlock": 0,
"eip150Block": 0,
"eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"eip155Block": 0,
"eip158Block": 0,
"byzantiumBlock": 0,
"constantinopleBlock": 0,
"petersburgBlock": 0,
"istanbulBlock": 0,
"clique": {
"period": 5,
"epoch": 30000
}
}
```
Below are the steps utilized to set up the test environment.
Provide the path to the above file in the config.
1. Run the `scripts/run_integration_test.sh` script.
1. First comment outline 130 to 133 from `validator_test/validator_test.go`
2. Once the code has completed running, comment out lines 55 to 126, 38 to 40, and 42 to 44.
1. Make the following change `db, err = setupDB() --> db, _ = setupDB()`
3. Run the following command: `ginkgo -r validator_test/ -v`
1. All tests should pass
## Usage
## Testing Failures
* Create / update the config file (refer to example config above).
Once we had populated the database, we tested for failures.
* Run validator:
```bash
./ipld-eth-db-validator stateValidator --config=<config path>
```
Example:
```bash
./ipld-eth-db-validator stateValidator --config=environments/example.toml
```
## Monitoring
* Enable metrics using config parameters `prom.metrics` and `prom.http`.
* `ipld-eth-db-validator` exposes following prometheus metrics at `/metrics` endpoint:
* `last_validated_block`: Last validated block number.
* DB stats if `prom.dbStats` set to `true`.
## Tests
* Follow [Test Instructions](./test/README.md) to run unit and integration tests locally.
1. Removing a Transaction from `transaction_cids` - If we removed a transaction from the database and ran the test, the test would fail. **This is the expected behavior.**
2. Removing Headers from `eth.header_cids`
1. If we removed a header block sandwiched between two header blocks, the test would fail (For example, we removed the entry for block 4, and the block range is 1-10). **This is the expected behavior.**
2. If we removed the tail block(s) from the table, the test would pass (For example, we remove the entry for blocks 8, 9, 10, and the block range is 1-10). **This is _not_ the expected behavior.**

View File

@ -1,84 +0,0 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"github.com/spf13/viper"
)
const (
LOG_LEVEL = "LOG_LEVEL"
LOG_FILE_PATH = "LOG_FILE_PATH"
PROM_METRICS = "PROM_METRICS"
PROM_HTTP = "PROM_HTTP"
PROM_HTTP_ADDR = "PROM_HTTP_ADDR"
PROM_HTTP_PORT = "PROM_HTTP_PORT"
PROM_DB_STATS = "PROM_DB_STATS"
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
ETH_CHAIN_CONFIG = "ETH_CHAIN_CONFIG"
ETH_CHAIN_ID = "ETH_CHAIN_ID"
ETH_HTTP_PATH = "ETH_HTTP_PATH"
VALIDATE_FROM_BLOCK = "VALIDATE_FROM_BLOCK"
VALIDATE_TRAIL = "VALIDATE_TRAIL"
VALIDATE_RETRY_INTERVAL = "VALIDATE_RETRY_INTERVAL"
VALIDATE_STATEDIFF_MISSING_BLOCK = "VALIDATE_STATEDIFF_MISSING_BLOCK"
VALIDATE_STATEDIFF_TIMEOUT = "VALIDATE_STATEDIFF_TIMEOUT"
)
// Bind env vars
func init() {
viper.BindEnv("log.level", LOG_LEVEL)
viper.BindEnv("log.file", LOG_FILE_PATH)
viper.BindEnv("prom.metrics", PROM_METRICS)
viper.BindEnv("prom.http", PROM_HTTP)
viper.BindEnv("prom.httpAddr", PROM_HTTP_ADDR)
viper.BindEnv("prom.httpPort", PROM_HTTP_PORT)
viper.BindEnv("prom.dbStats", PROM_DB_STATS)
viper.BindEnv("database.name", DATABASE_NAME)
viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
viper.BindEnv("database.port", DATABASE_PORT)
viper.BindEnv("database.user", DATABASE_USER)
viper.BindEnv("database.password", DATABASE_PASSWORD)
viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
viper.BindEnv("database.maxLifetime", DATABASE_MAX_CONN_LIFETIME)
viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
viper.BindEnv("ethereum.httpPath", ETH_HTTP_PATH)
viper.BindEnv("validate.fromBlock", VALIDATE_FROM_BLOCK)
viper.BindEnv("validate.trail", VALIDATE_TRAIL)
viper.BindEnv("validate.retryInterval", VALIDATE_RETRY_INTERVAL)
viper.BindEnv("validate.stateDiffMissingBlock", VALIDATE_STATEDIFF_MISSING_BLOCK)
viper.BindEnv("validate.stateDiffTimeout", VALIDATE_STATEDIFF_TIMEOUT)
}

View File

@ -1,31 +1,12 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"os"
"strings"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/prom"
)
var (
@ -50,21 +31,23 @@ func Execute() {
}
func initFunc(cmd *cobra.Command, args []string) {
ParseLogFlags()
if viper.GetBool("prom.metrics") {
log.Info("initializing prometheus metrics")
prom.Init()
logfile := viper.GetString("logfile")
if logfile != "" {
file, err := os.OpenFile(logfile,
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err == nil {
log.Infof("Directing output to %s", logfile)
log.SetOutput(file)
} else {
log.SetOutput(os.Stdout)
log.Info("Failed to log to file, using default stdout")
}
} else {
log.SetOutput(os.Stdout)
}
if viper.GetBool("prom.http") {
addr := fmt.Sprintf(
"%s:%s",
viper.GetString("prom.httpAddr"),
viper.GetString("prom.httpPort"),
)
log.Info("starting prometheus server")
prom.Serve(addr)
if err := logLevel(); err != nil {
log.Fatal("Could not set log level: ", err)
}
}
@ -74,31 +57,32 @@ func init() {
viper.AutomaticEnv()
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
rootCmd.PersistentFlags().String("database-name", "cerc_public", "database name")
rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
rootCmd.PersistentFlags().String("database-user", "", "database user")
rootCmd.PersistentFlags().String("database-password", "", "database password")
rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
rootCmd.PersistentFlags().Bool("prom-metrics", false, "enable prometheus metrics")
rootCmd.PersistentFlags().Bool("prom-http", false, "enable prometheus http service")
rootCmd.PersistentFlags().String("prom-httpAddr", "127.0.0.1", "prometheus http host")
rootCmd.PersistentFlags().String("prom-httpPort", "9001", "prometheus http port")
rootCmd.PersistentFlags().Bool("prom-dbStats", false, "enables prometheus db stats")
_ = viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
_ = viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
_ = viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
_ = viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
_ = viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
_ = viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
_ = viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
_ = viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
_ = viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("prom-metrics"))
_ = viper.BindPFlag("prom.http", rootCmd.PersistentFlags().Lookup("prom-http"))
_ = viper.BindPFlag("prom.httpAddr", rootCmd.PersistentFlags().Lookup("prom-httpAddr"))
_ = viper.BindPFlag("prom.httpPort", rootCmd.PersistentFlags().Lookup("prom-httpPort"))
_ = viper.BindPFlag("prom.dbStats", rootCmd.PersistentFlags().Lookup("prom-dbStats"))
}
func logLevel() error {
lvl, err := log.ParseLevel(viper.GetString("log.level"))
if err != nil {
return err
}
log.SetLevel(lvl)
if lvl > log.InfoLevel {
log.SetReportCaller(true)
}
log.Info("Log level set to ", lvl.String())
return nil
}

View File

@ -1,19 +1,3 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
@ -27,7 +11,7 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/validator"
"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
)
// stateValidatorCmd represents the stateValidator command
@ -49,10 +33,7 @@ func stateValidator() {
logWithCommand.Fatal(err)
}
service, err := validator.NewService(cfg, nil)
if err != nil {
logWithCommand.Fatal(err)
}
service := validator.NewService(cfg, nil)
wg := new(sync.WaitGroup)
wg.Add(1)
@ -68,25 +49,17 @@ func stateValidator() {
func init() {
rootCmd.AddCommand(stateValidatorCmd)
stateValidatorCmd.PersistentFlags().String("from-block", "1", "block height to initiate state validation")
stateValidatorCmd.PersistentFlags().String("trail", "64", "trail of block height to validate")
stateValidatorCmd.PersistentFlags().String("retry-interval", "10s", "retry interval in seconds after validator has caught up to (head-trail) height")
stateValidatorCmd.PersistentFlags().Bool("statediff-missing-block", false, "whether to perform a statediffing call on a missing block")
stateValidatorCmd.PersistentFlags().String("statediff-timeout", "240s", "statediffing call timeout period (in sec)")
stateValidatorCmd.PersistentFlags().String("block-height", "1", "block height to initiate state validation")
stateValidatorCmd.PersistentFlags().String("trail", "16", "trail of block height to validate")
stateValidatorCmd.PersistentFlags().String("sleep-interval", "10", "sleep interval in seconds after validator has caught up to (head-trail) height")
stateValidatorCmd.PersistentFlags().String("eth-chain-config", "", "path to json chain config")
stateValidatorCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
stateValidatorCmd.PersistentFlags().String("eth-http-path", "", "http url for a statediffing node")
stateValidatorCmd.PersistentFlags().String("chain-config", "", "path to chain config")
_ = viper.BindPFlag("validate.fromBlock", stateValidatorCmd.PersistentFlags().Lookup("from-block"))
_ = viper.BindPFlag("validate.block-height", stateValidatorCmd.PersistentFlags().Lookup("block-height"))
_ = viper.BindPFlag("validate.trail", stateValidatorCmd.PersistentFlags().Lookup("trail"))
_ = viper.BindPFlag("validate.retryInterval", stateValidatorCmd.PersistentFlags().Lookup("retry-interval"))
_ = viper.BindPFlag("validate.stateDiffMissingBlock", stateValidatorCmd.PersistentFlags().Lookup("statediff-missing-block"))
_ = viper.BindPFlag("validate.stateDiffTimeout", stateValidatorCmd.PersistentFlags().Lookup("statediff-timeout"))
_ = viper.BindPFlag("validate.sleepInterval", stateValidatorCmd.PersistentFlags().Lookup("sleep-interval"))
_ = viper.BindPFlag("ethereum.chainConfig", stateValidatorCmd.PersistentFlags().Lookup("eth-chain-config"))
_ = viper.BindPFlag("ethereum.chainID", stateValidatorCmd.PersistentFlags().Lookup("eth-chain-id"))
_ = viper.BindPFlag("ethereum.httpPath", stateValidatorCmd.PersistentFlags().Lookup("eth-http-path"))
_ = viper.BindPFlag("ethereum.chainConfig", stateValidatorCmd.PersistentFlags().Lookup("chain-config"))
}
func initConfig() {

View File

@ -1,35 +0,0 @@
package cmd
import (
"os"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
func ParseLogFlags() {
logfile := viper.GetString("log.file")
if logfile != "" {
file, err := os.OpenFile(logfile,
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err == nil {
log.Infof("Directing output to %s", logfile)
log.SetOutput(file)
} else {
log.SetOutput(os.Stdout)
log.Info("Failed to log to file, using default stdout")
}
} else {
log.SetOutput(os.Stdout)
}
lvl, err := log.ParseLevel(viper.GetString("log.level"))
if err != nil {
log.Fatal("Could not parse log level: ", err)
}
log.SetLevel(lvl)
if lvl > log.InfoLevel {
log.SetReportCaller(true)
}
log.Info("Log level set to ", lvl)
}

View File

@ -1,26 +1,30 @@
# Containers to run backing DB for unit testing
version: '3.2'
services:
migrations:
restart: on-failure
depends_on:
- ipld-eth-db
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.3.0-alpha
image: vulcanize/ipld-eth-db:v4.2.0-alpha
environment:
DATABASE_USER: "vdbm"
DATABASE_NAME: "cerc_testing"
DATABASE_NAME: "vulcanize_testing"
DATABASE_PASSWORD: "password"
DATABASE_HOSTNAME: "ipld-eth-db"
DATABASE_PORT: 5432
ipld-eth-db:
container_name: test-ipld-eth-db
image: timescale/timescaledb:latest-pg14
restart: always
command: ["postgres", "-c", "log_statement=all"]
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "cerc_testing"
POSTGRES_DB: "vulcanize_testing"
POSTGRES_PASSWORD: "password"
ports:
- "127.0.0.1:8077:5432"
volumes:
- vdb_db_eth_validator:/var/lib/postgresql/data
volumes:
vdb_db_eth_validator:

View File

@ -1,30 +1,14 @@
[database]
name = "cerc_public"
name = "vulcanize_public"
hostname = "localhost"
port = 5432
password = "password"
user = "vdbm"
[validate]
fromBlock = 1
trail = 64
retryInterval = "10s"
stateDiffMissingBlock = true
stateDiffTimeout = "240s"
block-height = 1
trail = 16
sleepInterval = 10
[ethereum]
chainConfig = ""
chainID = "1"
httpPath = "localhost:8545"
wsPath = "localhost:8546"
[prom]
metrics = true
http = true
httpAddr = "localhost"
httpPort = "9001"
dbStats = true
[log]
file = ""
level = "info"
chainConfig = "./chain.json"

440
go.mod
View File

@ -1,274 +1,278 @@
module github.com/cerc-io/ipld-eth-db-validator/v5
module github.com/vulcanize/ipld-eth-db-validator
go 1.21
go 1.18
require (
github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha
github.com/cerc-io/ipld-eth-server/v5 v5.3.0-alpha
github.com/cerc-io/ipld-eth-statedb v0.1.1
github.com/cerc-io/plugeth-statediff v0.3.2
github.com/ethereum/go-ethereum v1.13.14
github.com/holiman/uint256 v1.2.4
github.com/ethereum/go-ethereum v1.10.19
github.com/jmoiron/sqlx v1.3.5
github.com/onsi/ginkgo/v2 v2.15.0
github.com/onsi/gomega v1.30.0
github.com/prometheus/client_golang v1.18.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.18.2
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.19.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.4.0
github.com/spf13/viper v1.11.0
github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha
github.com/vulcanize/ipld-eth-server/v4 v4.1.0-alpha
)
require (
bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect
github.com/DataDog/zstd v1.5.5 // indirect
github.com/Jorropo/jsync v1.0.1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
github.com/Stebalien/go-bitfield v0.0.1 // indirect
github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.10.0 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha // indirect
github.com/cerc-io/eth-iterator-utils v0.3.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cockroachdb/errors v1.10.0 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/consensys/bavard v0.1.13 // indirect
github.com/consensys/gnark-crypto v0.12.1 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
github.com/btcsuite/btcd v0.22.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/cskr/pubsub v1.0.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/deckarep/golang-set/v2 v2.3.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
github.com/deckarep/golang-set v1.8.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/edsrzf/mmap-go v1.0.0 // indirect
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
github.com/flynn/noise v1.1.0 // indirect
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect
github.com/georgysavva/scany v0.2.9 // indirect
github.com/getsentry/sentry-go v0.22.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-stack/stack v1.8.1 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-bexpr v0.1.10 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
github.com/huin/goupnp v1.3.0 // indirect
github.com/inconshreveable/log15 v2.16.0+incompatible // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/holiman/uint256 v1.2.0 // indirect
github.com/huin/goupnp v1.0.3 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/boxo v0.19.0 // indirect
github.com/ipfs/go-bitfield v1.1.0 // indirect
github.com/ipfs/go-block-format v0.2.0 // indirect
github.com/ipfs/go-cid v0.4.1 // indirect
github.com/ipfs/go-cidutil v0.1.0 // indirect
github.com/ipfs/go-datastore v0.6.0 // indirect
github.com/ipfs/go-ds-measure v0.2.0 // indirect
github.com/ipfs/go-bitswap v0.4.0 // indirect
github.com/ipfs/go-block-format v0.0.3 // indirect
github.com/ipfs/go-blockservice v0.1.7 // indirect
github.com/ipfs/go-cid v0.0.7 // indirect
github.com/ipfs/go-cidutil v0.0.2 // indirect
github.com/ipfs/go-datastore v0.4.6 // indirect
github.com/ipfs/go-ds-measure v0.1.0 // indirect
github.com/ipfs/go-fetcher v1.5.0 // indirect
github.com/ipfs/go-filestore v1.0.0 // indirect
github.com/ipfs/go-fs-lock v0.0.7 // indirect
github.com/ipfs/go-graphsync v0.8.0 // indirect
github.com/ipfs/go-ipfs v0.10.0 // indirect
github.com/ipfs/go-ipfs-blockstore v1.0.1 // indirect
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
github.com/ipfs/go-ipfs-config v0.16.0 // indirect
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
github.com/ipfs/go-ipfs-util v0.0.3 // indirect
github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
github.com/ipfs/go-ipld-format v0.6.0 // indirect
github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
github.com/ipfs/go-ipfs-ds-help v1.0.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.0.1 // indirect
github.com/ipfs/go-ipfs-exchange-offline v0.0.1 // indirect
github.com/ipfs/go-ipfs-files v0.0.8 // indirect
github.com/ipfs/go-ipfs-keystore v0.0.2 // indirect
github.com/ipfs/go-ipfs-pinner v0.1.2 // indirect
github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
github.com/ipfs/go-ipfs-provider v0.6.1 // indirect
github.com/ipfs/go-ipfs-routing v0.1.0 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
github.com/ipfs/go-ipld-format v0.2.0 // indirect
github.com/ipfs/go-ipld-legacy v0.1.0 // indirect
github.com/ipfs/go-ipns v0.1.2 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-log/v2 v2.3.0 // indirect
github.com/ipfs/go-merkledag v0.4.0 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
github.com/ipfs/go-unixfsnode v1.9.0 // indirect
github.com/ipfs/kubo v0.27.0 // indirect
github.com/ipld/go-car/v2 v2.13.1 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/ipld/go-ipld-prime v0.21.0 // indirect
github.com/ipfs/go-mfs v0.1.2 // indirect
github.com/ipfs/go-namesys v0.3.1 // indirect
github.com/ipfs/go-path v0.1.2 // indirect
github.com/ipfs/go-peertaskqueue v0.4.0 // indirect
github.com/ipfs/go-unixfs v0.2.5 // indirect
github.com/ipfs/go-unixfsnode v1.1.3 // indirect
github.com/ipfs/go-verifcid v0.0.1 // indirect
github.com/ipfs/interface-go-ipfs-core v0.5.1 // indirect
github.com/ipld/go-codec-dagpb v1.3.0 // indirect
github.com/ipld/go-ipld-prime v0.12.2 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.14.3 // indirect
github.com/jackc/pgconn v1.12.1 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.3 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgtype v1.14.0 // indirect
github.com/jackc/pgx/v4 v4.18.3 // indirect
github.com/jackc/puddle v1.3.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/pgtype v1.11.0 // indirect
github.com/jackc/pgx/v4 v4.16.1 // indirect
github.com/jackc/puddle v1.2.1 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.4 // indirect
github.com/klauspost/compress v1.17.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/klauspost/compress v1.11.7 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/koron/go-ssdp v0.0.2 // indirect
github.com/lib/pq v1.10.6 // indirect
github.com/libp2p/go-addr-util v0.1.0 // indirect
github.com/libp2p/go-buffer-pool v0.0.2 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-doh-resolver v0.4.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
github.com/libp2p/go-libp2p v0.33.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.24.4 // indirect
github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
github.com/libp2p/go-libp2p-pubsub v0.10.0 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.3 // indirect
github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
github.com/libp2p/go-msgio v0.3.0 // indirect
github.com/libp2p/go-nat v0.2.0 // indirect
github.com/libp2p/go-netroute v0.2.1 // indirect
github.com/libp2p/go-reuseport v0.4.0 // indirect
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/libp2p/go-conn-security-multistream v0.2.1 // indirect
github.com/libp2p/go-doh-resolver v0.3.1 // indirect
github.com/libp2p/go-eventbus v0.2.1 // indirect
github.com/libp2p/go-flow-metrics v0.0.3 // indirect
github.com/libp2p/go-libp2p v0.15.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 // indirect
github.com/libp2p/go-libp2p-autonat v0.4.2 // indirect
github.com/libp2p/go-libp2p-blankhost v0.2.0 // indirect
github.com/libp2p/go-libp2p-circuit v0.4.0 // indirect
github.com/libp2p/go-libp2p-connmgr v0.2.4 // indirect
github.com/libp2p/go-libp2p-core v0.9.0 // indirect
github.com/libp2p/go-libp2p-discovery v0.5.1 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.13.1 // indirect
github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect
github.com/libp2p/go-libp2p-loggables v0.1.0 // indirect
github.com/libp2p/go-libp2p-mplex v0.4.1 // indirect
github.com/libp2p/go-libp2p-nat v0.0.6 // indirect
github.com/libp2p/go-libp2p-noise v0.2.2 // indirect
github.com/libp2p/go-libp2p-peerstore v0.2.8 // indirect
github.com/libp2p/go-libp2p-pnet v0.2.0 // indirect
github.com/libp2p/go-libp2p-pubsub v0.5.4 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.4.0 // indirect
github.com/libp2p/go-libp2p-quic-transport v0.12.0 // indirect
github.com/libp2p/go-libp2p-record v0.1.3 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.2.3 // indirect
github.com/libp2p/go-libp2p-swarm v0.5.3 // indirect
github.com/libp2p/go-libp2p-tls v0.2.0 // indirect
github.com/libp2p/go-libp2p-transport-upgrader v0.4.6 // indirect
github.com/libp2p/go-libp2p-xor v0.0.0-20210714161855-5c005aca55db // indirect
github.com/libp2p/go-libp2p-yamux v0.5.4 // indirect
github.com/libp2p/go-maddr-filter v0.1.0 // indirect
github.com/libp2p/go-mplex v0.3.0 // indirect
github.com/libp2p/go-msgio v0.0.6 // indirect
github.com/libp2p/go-nat v0.0.5 // indirect
github.com/libp2p/go-netroute v0.1.6 // indirect
github.com/libp2p/go-openssl v0.0.7 // indirect
github.com/libp2p/go-reuseport v0.0.2 // indirect
github.com/libp2p/go-reuseport-transport v0.0.5 // indirect
github.com/libp2p/go-sockaddr v0.1.1 // indirect
github.com/libp2p/go-stream-muxer-multistream v0.3.0 // indirect
github.com/libp2p/go-tcp-transport v0.2.8 // indirect
github.com/libp2p/go-ws-transport v0.5.0 // indirect
github.com/libp2p/go-yamux/v2 v2.2.0 // indirect
github.com/libp2p/zeroconf/v2 v2.0.0 // indirect
github.com/lucas-clemente/quic-go v0.26.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailgun/groupcache/v2 v2.3.0 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/miekg/dns v1.1.58 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/miekg/dns v1.1.43 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mitchellh/pointerstructure v1.2.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multiaddr v0.12.2 // indirect
github.com/multiformats/go-base32 v0.0.3 // indirect
github.com/multiformats/go-base36 v0.1.0 // indirect
github.com/multiformats/go-multiaddr v0.4.0 // indirect
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multibase v0.2.0 // indirect
github.com/multiformats/go-multicodec v0.9.0 // indirect
github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-multistream v0.5.0 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/multiformats/go-multibase v0.0.3 // indirect
github.com/multiformats/go-multicodec v0.3.0 // indirect
github.com/multiformats/go-multihash v0.1.0 // indirect
github.com/multiformats/go-multistream v0.2.2 // indirect
github.com/multiformats/go-varint v0.0.6 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opencontainers/runtime-spec v1.2.0 // indirect
github.com/openrelayxyz/plugeth-utils v1.5.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
github.com/pganalyze/pg_query_go/v4 v4.2.1 // indirect
github.com/pion/datachannel v1.5.5 // indirect
github.com/pion/dtls/v2 v2.2.8 // indirect
github.com/pion/ice/v2 v2.3.11 // indirect
github.com/pion/interceptor v0.1.25 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns v0.0.9 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.13 // indirect
github.com/pion/rtp v1.8.3 // indirect
github.com/pion/sctp v1.8.9 // indirect
github.com/pion/sdp/v3 v3.0.6 // indirect
github.com/pion/srtp/v2 v2.0.18 // indirect
github.com/pion/stun v0.6.1 // indirect
github.com/pion/transport/v2 v2.2.4 // indirect
github.com/pion/turn/v2 v2.1.4 // indirect
github.com/pion/webrtc/v3 v3.2.23 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.47.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/quic-go v0.41.0 // indirect
github.com/quic-go/webtransport-go v0.6.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/samber/lo v1.39.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.30.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/prometheus/tsdb v0.7.1 // indirect
github.com/rjeczalik/notify v0.9.1 // indirect
github.com/rs/cors v1.7.0 // indirect
github.com/segmentio/fasthash v1.0.3 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/shirou/gopsutil v3.21.5+incompatible // indirect
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/status-im/keycard-go v0.2.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/stretchr/testify v1.9.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/supranational/blst v0.3.11 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
github.com/thoas/go-funk v0.9.3 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/stretchr/testify v1.7.1 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/thoas/go-funk v0.9.2 // indirect
github.com/tklauser/go-sysconf v0.3.6 // indirect
github.com/tklauser/numcpus v0.2.2 // indirect
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.3-alpha // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20240109153615-66e95c3e8a87 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 // indirect
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.25.0 // indirect
go.opentelemetry.io/otel/metric v1.25.0 // indirect
go.opentelemetry.io/otel/trace v1.25.0 // indirect
go.uber.org/dig v1.17.1 // indirect
go.uber.org/fx v1.20.1 // indirect
go.uber.org/mock v0.4.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.20.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
gonum.org/v1/gonum v0.14.0 // indirect
google.golang.org/protobuf v1.32.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/otel v0.20.0 // indirect
go.opentelemetry.io/otel/metric v0.20.0 // indirect
go.opentelemetry.io/otel/trace v0.20.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/dig v1.10.0 // indirect
go.uber.org/fx v1.13.1 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.19.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
golang.org/x/tools v0.1.8 // indirect
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/urfave/cli.v1 v1.20.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gorm.io/driver/postgres v1.3.7 // indirect
gorm.io/gorm v1.23.5 // indirect
lukechampine.com/blake3 v1.2.2 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
lukechampine.com/blake3 v1.1.6 // indirect
)
replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha

1983
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -1,64 +0,0 @@
package integration
import (
"encoding/json"
"fmt"
"math/big"
"net/http"
)
type ContractDeployed struct {
Address string `json:"address"`
TransactionHash string `json:"txHash"`
BlockNumber uint64 `json:"blockNumber"`
BlockHash string `json:"blockHash"`
}
type ContractDestroyed struct {
BlockNumber uint64 `json:"blockNumber"`
}
type PutResult struct {
BlockNumber uint64 `json:"blockNumber"`
}
type Tx struct {
From string `json:"from"`
To string `json:"to"`
Value *big.Int `json:"value"`
TransactionHash string `json:"txHash"`
BlockNumber uint64 `json:"blockNumber"`
BlockHash string `json:"blockHash"`
}
const ContractServerUrl = "http://localhost:3000"
// Factory to generate endpoint functions
func MakeGetAndDecodeFunc[R any](format string) func(...interface{}) (*R, error) {
return func(params ...interface{}) (*R, error) {
params = append([]interface{}{ContractServerUrl}, params...)
url := fmt.Sprintf(format, params...)
res, err := http.Get(url)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s: %s", url, res.Status)
}
var data R
decoder := json.NewDecoder(res.Body)
return &data, decoder.Decode(&data)
}
}
var (
SendEth = MakeGetAndDecodeFunc[Tx]("%s/v1/sendEth?to=%s&value=%s")
DeployContract = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/deployContract")
DestroyContract = MakeGetAndDecodeFunc[ContractDestroyed]("%s/v1/destroyContract?addr=%s")
DeployTestContract = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/deployTestContract")
DestroyTestContract = MakeGetAndDecodeFunc[ContractDestroyed]("%s/v1/destroyTestContract?addr=%s")
PutTestValue = MakeGetAndDecodeFunc[PutResult]("%s/v1/putTestValue?addr=%s&value=%d")
)

View File

@ -1,140 +0,0 @@
package integration_test
import (
"context"
"sync"
"testing"
"time"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/onsi/gomega"
. "github.com/onsi/gomega"
"github.com/cerc-io/ipld-eth-db-validator/v5/cmd" // this registers env vars with viper
"github.com/cerc-io/ipld-eth-db-validator/v5/integration"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/validator"
)
const (
timeout = 20 * time.Minute
pollInterval = time.Second
progressBufferSize = 200
)
var (
testAddresses = []string{
"0x1111111111111111111111111111111111111112",
"0x1ca7c995f8eF0A2989BbcE08D5B7Efe50A584aa1",
"0x9a4b666af23a2cdb4e5538e1d222a445aeb82134",
"0xF7C7AEaECD2349b129d5d15790241c32eeE4607B",
"0x992b6E9BFCA1F7b0797Cee10b0170E536EAd3532",
}
// Track the blocks validated on this chain
lastValidated uint64
validated = newBlockSet()
ctx = context.Background()
wg sync.WaitGroup
)
func init() {
cmd.ParseLogFlags()
gomega.SetDefaultEventuallyTimeout(timeout)
gomega.SetDefaultEventuallyPollingInterval(pollInterval)
}
func setup(t *testing.T, progressChan chan uint64) {
cfg, err := validator.NewConfig()
if err != nil {
t.Fatal(err)
}
// set the default DB config to the testing defaults
cfg.DBConfig, _ = postgres.TestConfig.WithEnv()
// update the start block if we have already validated past it
if lastValidated > cfg.FromBlock {
cfg.FromBlock = lastValidated
}
// default trail is unnecessarily long
cfg.Trail = 8
service, err := validator.NewService(cfg, progressChan)
if err != nil {
t.Fatal(err)
}
go func() {
for block := range progressChan {
validated.add(block)
lastValidated = block
}
}()
wg.Add(1)
go service.Start(ctx, &wg)
t.Cleanup(func() {
service.Stop()
wg.Wait()
g := gomega.NewWithT(t)
g.Expect(progressChan).To(BeClosed())
})
}
func TestValidateContracts(t *testing.T) {
progressChan := make(chan uint64, progressBufferSize)
setup(t, progressChan)
contract, err := integration.DeployTestContract()
if err != nil {
t.Fatal(err)
}
t.Run("contract deployment", func(t *testing.T) {
g := gomega.NewWithT(t)
t.Logf("Deployed contract at block %d", contract.BlockNumber)
g.Expect(progressChan).ToNot(BeClosed())
g.Eventually(validated.contains, timeout).WithArguments(contract.BlockNumber).Should(BeTrue())
})
t.Run("contract method calls", func(t *testing.T) {
g := gomega.NewWithT(t)
var blocks []uint64
for i := 0; i < 3; i++ {
res, err := integration.PutTestValue(contract.Address, i)
if err != nil {
t.Fatal(err)
}
t.Logf("Put() called at block %d", res.BlockNumber)
blocks = append(blocks, res.BlockNumber)
}
g.Expect(progressChan).ToNot(BeClosed())
g.Eventually(validated.containsAll, timeout).WithArguments(blocks).Should(BeTrue())
})
}
func TestValidateTransactions(t *testing.T) {
progressChan := make(chan uint64, progressBufferSize)
setup(t, progressChan)
t.Run("ETH transfer transactions", func(t *testing.T) {
g := gomega.NewWithT(t)
var blocks []uint64
for _, address := range testAddresses {
tx, err := integration.SendEth(address, "0.01")
if err != nil {
t.Fatal(err)
}
t.Logf("Sent tx at block %d", tx.BlockNumber)
blocks = append(blocks, tx.BlockNumber)
}
g.Expect(progressChan).ToNot(BeClosed())
g.Eventually(validated.containsAll, timeout).WithArguments(blocks).Should(BeTrue())
})
}

View File

@ -1,36 +0,0 @@
package integration_test
import (
"sync"
)
type atomicBlockSet struct {
blocks map[uint64]struct{}
sync.RWMutex
}
func newBlockSet() *atomicBlockSet {
return &atomicBlockSet{blocks: make(map[uint64]struct{})}
}
func (set *atomicBlockSet) contains(block uint64) bool {
set.RLock()
defer set.RUnlock()
_, has := set.blocks[block]
return has
}
func (set *atomicBlockSet) containsAll(blocks []uint64) bool {
for _, block := range blocks {
if !set.contains(block) {
return false
}
}
return true
}
func (set *atomicBlockSet) add(block uint64) {
set.Lock()
defer set.Unlock()
set.blocks[block] = struct{}{}
}

View File

@ -1,100 +0,0 @@
package chaingen
import (
"math/big"
"github.com/cerc-io/plugeth-statediff/test_helpers"
"github.com/cerc-io/plugeth-statediff/test_helpers/chaingen"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/cerc-io/ipld-eth-db-validator/v5/internal/testdata"
)
var (
bank, acct1, acct2 common.Address
contractAddr common.Address
contractDataRoot string
defaultContract *chaingen.ContractSpec
)
func init() {
var err error
defaultContract, err = chaingen.ParseContract(testdata.TestContractABI, testdata.TestContractCode)
if err != nil {
panic(err)
}
}
// A GenContext which exactly replicates the chain generator used in existing tests
func DefaultGenContext(chainConfig *params.ChainConfig, db ethdb.Database) *chaingen.GenContext {
gen := chaingen.NewGenContext(chainConfig, db)
bank = gen.AddOwnedAccount(test_helpers.TestBankKey)
acct1 = gen.AddOwnedAccount(test_helpers.Account1Key)
acct2 = gen.AddOwnedAccount(test_helpers.Account2Key)
gen.AddContract("Test", defaultContract)
gen.AddFunction(func(i int, block *core.BlockGen) {
if err := defaultChainGen(gen, i, block); err != nil {
panic(err)
}
})
gen.Genesis = test_helpers.GenesisBlockForTesting(
db, bank, test_helpers.TestBankFunds, big.NewInt(params.InitialBaseFee), params.MaxGasLimit,
)
return gen
}
func defaultChainGen(gen *chaingen.GenContext, i int, block *core.BlockGen) error {
switch i {
case 0:
// In block 1, the test bank sends account #1 some ether.
tx, err := gen.CreateSendTx(bank, acct1, big.NewInt(10000))
if err != nil {
panic(err)
}
block.AddTx(tx)
case 1:
// In block 2, the test bank sends some more ether to account #1.
// acct1 passes it on to account #2.
// acct1 creates a test contract.
tx1, err := gen.CreateSendTx(bank, acct1, big.NewInt(1000))
if err != nil {
panic(err)
}
block.AddTx(tx1)
tx2, err := gen.CreateSendTx(acct1, acct2, big.NewInt(1000))
if err != nil {
panic(err)
}
block.AddTx(tx2)
contractAddr, err = gen.DeployContract(acct1, "Test")
if err != nil {
panic(err)
}
case 2:
block.SetCoinbase(acct2)
tx, err := gen.CreateCallTx(bank, contractAddr, "Put", big.NewInt(3))
if err != nil {
panic(err)
}
block.AddTx(tx)
case 3:
block.SetCoinbase(acct2)
tx, err := gen.CreateCallTx(bank, contractAddr, "Put", big.NewInt(9))
if err != nil {
panic(err)
}
block.AddTx(tx)
case 4:
block.SetCoinbase(acct1)
tx, err := gen.CreateCallTx(bank, contractAddr, "Put", big.NewInt(0))
if err != nil {
panic(err)
}
block.AddTx(tx)
}
return nil
}

View File

@ -1 +0,0 @@
[{"inputs":[],"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"address","name":"","type":"address"},{"indexed":false,"internalType":"uint256","name":"","type":"uint256"}],"name":"logPut","type":"event"},{"inputs":[{"internalType":"uint256","name":"value","type":"uint256"}],"name":"Put","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"close","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"data","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"}]

View File

@ -1 +0,0 @@
608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550610459806100606000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a14610050578063b90d3d0c1461006c575b600080fd5b61004e61009c565b005b61006a60048036038101906100659190610266565b610193565b005b610086600480360381019061008191906102f1565b610213565b604051610093919061032d565b60405180910390f35b60008054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161461012a576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610121906103cb565b60405180910390fd5b60008054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff166108fc479081150290604051600060405180830381858888f19350505050158015610190573d6000803e3d6000fd5b50565b7f370acc53a76362ca0f71a1b2e0c8b8ffbbc1ba9ff3166a1e2fa8445b4848626c33826040516101c49291906103fa565b60405180910390a180600160003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000208190555050565b60016020528060005260406000206000915090505481565b600080fd5b6000819050919050565b61024381610230565b811461024e57600080fd5b50565b6000813590506102608161023a565b92915050565b60006020828403121561027c5761027b61022b565b5b600061028a84828501610251565b91505092915050565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006102be82610293565b9050919050565b6102ce816102b3565b81146102d957600080fd5b50565b6000813590506102eb816102c5565b92915050565b6000602082840312156103075761030661022b565b5b6000610315848285016102dc565b91505092915050565b61032781610230565b82525050565b6000602082019050610342600083018461031e565b92915050565b600082825260208201905092915050565b7f4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f60008201527f6e2e000000000000000000000000000000000000000000000000000000000000602082015250565b60006103b5602283610348565b91506103c082610359565b604082019050919050565b600060208201905081810360008301526103e4816103a8565b9050919050565b6103f4816102b3565b82525050565b600060408201905061040f60008301856103eb565b61041c602083018461031e565b939250505056fea26469706673582212205c59738e694ff2e1f03f21fa2e7d943137128c0f43ae0e8175a589b32fa81a9a64736f6c63430008140033

View File

@ -1,10 +0,0 @@
package testdata
import _ "embed"
var (
//go:embed build/Test.abi
TestContractABI string
//go:embed build/Test.bin
TestContractCode string
)

18
main.go
View File

@ -1,22 +1,6 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import "github.com/cerc-io/ipld-eth-db-validator/v5/cmd"
import "github.com/vulcanize/ipld-eth-db-validator/cmd"
func main() {
cmd.Execute()

View File

@ -1,157 +0,0 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package prom
import (
"database/sql"
"github.com/prometheus/client_golang/prometheus"
)
// DBStatsGetter is an interface that gets sql.DBStats.
type DBStatsGetter interface {
Stats() sql.DBStats
}
// DBStatsCollector implements the prometheus.Collector interface.
type DBStatsCollector struct {
sg DBStatsGetter
// descriptions of exported metrics
maxOpenDesc *prometheus.Desc
openDesc *prometheus.Desc
inUseDesc *prometheus.Desc
idleDesc *prometheus.Desc
waitedForDesc *prometheus.Desc
blockedSecondsDesc *prometheus.Desc
closedMaxIdleDesc *prometheus.Desc
closedMaxLifetimeDesc *prometheus.Desc
}
// NewDBStatsCollector creates a new DBStatsCollector.
func NewDBStatsCollector(dbName string, sg DBStatsGetter) *DBStatsCollector {
labels := prometheus.Labels{"db_name": dbName}
return &DBStatsCollector{
sg: sg,
maxOpenDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "max_open"),
"Maximum number of open connections to the database.",
nil,
labels,
),
openDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "open"),
"The number of established connections both in use and idle.",
nil,
labels,
),
inUseDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "in_use"),
"The number of connections currently in use.",
nil,
labels,
),
idleDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "idle"),
"The number of idle connections.",
nil,
labels,
),
waitedForDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "waited_for"),
"The total number of connections waited for.",
nil,
labels,
),
blockedSecondsDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "blocked_seconds"),
"The total time blocked waiting for a new connection.",
nil,
labels,
),
closedMaxIdleDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "closed_max_idle"),
"The total number of connections closed due to SetMaxIdleConns.",
nil,
labels,
),
closedMaxLifetimeDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, connSubsystem, "closed_max_lifetime"),
"The total number of connections closed due to SetConnMaxLifetime.",
nil,
labels,
),
}
}
// Describe implements the prometheus.Collector interface.
func (c DBStatsCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.maxOpenDesc
ch <- c.openDesc
ch <- c.inUseDesc
ch <- c.idleDesc
ch <- c.waitedForDesc
ch <- c.blockedSecondsDesc
ch <- c.closedMaxIdleDesc
ch <- c.closedMaxLifetimeDesc
}
// Collect implements the prometheus.Collector interface.
func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) {
stats := c.sg.Stats()
ch <- prometheus.MustNewConstMetric(
c.maxOpenDesc,
prometheus.GaugeValue,
float64(stats.MaxOpenConnections),
)
ch <- prometheus.MustNewConstMetric(
c.openDesc,
prometheus.GaugeValue,
float64(stats.OpenConnections),
)
ch <- prometheus.MustNewConstMetric(
c.inUseDesc,
prometheus.GaugeValue,
float64(stats.InUse),
)
ch <- prometheus.MustNewConstMetric(
c.idleDesc,
prometheus.GaugeValue,
float64(stats.Idle),
)
ch <- prometheus.MustNewConstMetric(
c.waitedForDesc,
prometheus.CounterValue,
float64(stats.WaitCount),
)
ch <- prometheus.MustNewConstMetric(
c.blockedSecondsDesc,
prometheus.CounterValue,
stats.WaitDuration.Seconds(),
)
ch <- prometheus.MustNewConstMetric(
c.closedMaxIdleDesc,
prometheus.CounterValue,
float64(stats.MaxIdleClosed),
)
ch <- prometheus.MustNewConstMetric(
c.closedMaxLifetimeDesc,
prometheus.CounterValue,
float64(stats.MaxLifetimeClosed),
)
}

View File

@ -1,59 +0,0 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package prom
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
const (
namespace = "ipld_eth_state_snapshot"
connSubsystem = "connections"
statsSubsystem = "stats"
)
var (
metrics bool
lastValidatedBlock prometheus.Gauge
)
func Init() {
metrics = true
lastValidatedBlock = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "last_validated_block",
Help: "Last validated block number",
})
}
// RegisterDBCollector create metric collector for given connection
func RegisterDBCollector(name string, db DBStatsGetter) {
if metrics {
prometheus.Register(NewDBStatsCollector(name, db))
}
}
// SetLastValidatedBlock sets the last validated block number
func SetLastValidatedBlock(blockNumber float64) {
if metrics {
lastValidatedBlock.Set(blockNumber)
}
}

View File

@ -1,47 +0,0 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package prom
import (
"errors"
"net/http"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/sirupsen/logrus"
)
var errPromHTTP = errors.New("can't start http server for prometheus")
// Serve start listening http
func Serve(addr string) *http.Server {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
srv := http.Server{
Addr: addr,
Handler: mux,
}
go func() {
if err := srv.ListenAndServe(); err != nil {
logrus.
WithError(err).
WithField("module", "prom").
WithField("addr", addr).
Fatal(errPromHTTP)
}
}()
return &srv
}

View File

@ -1,43 +1,70 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
import (
"fmt"
"math/big"
"time"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/utils"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
)
type Config struct {
DBConfig postgres.Config
DBStats bool
var (
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
)
ChainConfig *params.ChainConfig
// Used to trigger writing state diffs for gaps in the index
Client *rpc.Client
FromBlock, Trail uint64
RetryInterval time.Duration
StateDiffMissingBlock bool
StateDiffTimeout time.Duration
var IntegrationTestChainConfig = &params.ChainConfig{
ChainID: big.NewInt(99),
HomesteadBlock: big.NewInt(0),
EIP150Block: big.NewInt(0),
EIP155Block: big.NewInt(0),
EIP158Block: big.NewInt(0),
ByzantiumBlock: big.NewInt(0),
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
Clique: &params.CliqueConfig{
Period: 0,
Epoch: 30000,
},
}
var TestChainConfig = &params.ChainConfig{
ChainID: big.NewInt(1),
HomesteadBlock: big.NewInt(0),
EIP150Block: big.NewInt(0),
EIP155Block: big.NewInt(0),
EIP158Block: big.NewInt(0),
ByzantiumBlock: big.NewInt(0),
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
MuirGlacierBlock: big.NewInt(0),
BerlinBlock: big.NewInt(0),
LondonBlock: big.NewInt(6),
ArrowGlacierBlock: big.NewInt(0),
Ethash: new(params.EthashConfig),
}
type Config struct {
dbConfig postgres.Config
DB *sqlx.DB
ChainCfg *params.ChainConfig
BlockNum, Trail uint64
SleepInterval uint
}
func NewConfig() (*Config, error) {
@ -47,12 +74,16 @@ func NewConfig() (*Config, error) {
return nil, err
}
err = cfg.setupEth()
if err != nil {
return nil, err
cfg.BlockNum = viper.GetUint64("validate.block-height")
if cfg.BlockNum < 1 {
return nil, fmt.Errorf("block height cannot be less the 1")
}
err = cfg.setupValidator()
cfg.Trail = viper.GetUint64("validate.trail")
cfg.SleepInterval = viper.GetUint("validate.sleepInterval")
chainConfigPath := viper.GetString("ethereum.chainConfig")
cfg.ChainCfg, err = statediff.LoadConfig(chainConfigPath)
if err != nil {
return nil, err
}
@ -61,59 +92,32 @@ func NewConfig() (*Config, error) {
}
func (c *Config) setupDB() error {
_ = viper.BindEnv("database.name", DATABASE_NAME)
_ = viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
_ = viper.BindEnv("database.port", DATABASE_PORT)
_ = viper.BindEnv("database.user", DATABASE_USER)
_ = viper.BindEnv("database.password", DATABASE_PASSWORD)
_ = viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
_ = viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
_ = viper.BindEnv("database.maxLifetime", DATABASE_MAX_CONN_LIFETIME)
// DB Config
c.DBConfig.DatabaseName = viper.GetString("database.name")
c.DBConfig.Hostname = viper.GetString("database.hostname")
c.DBConfig.Port = viper.GetInt("database.port")
c.DBConfig.Username = viper.GetString("database.user")
c.DBConfig.Password = viper.GetString("database.password")
c.dbConfig.DatabaseName = viper.GetString("database.name")
c.dbConfig.Hostname = viper.GetString("database.hostname")
c.dbConfig.Port = viper.GetInt("database.port")
c.dbConfig.Username = viper.GetString("database.user")
c.dbConfig.Password = viper.GetString("database.password")
c.DBConfig.MaxIdle = viper.GetInt("database.maxIdle")
c.DBConfig.MaxConns = viper.GetInt("database.maxOpen")
c.DBConfig.MaxConnLifetime = viper.GetDuration("database.maxLifetime")
c.dbConfig.MaxIdle = viper.GetInt("database.maxIdle")
c.dbConfig.MaxConns = viper.GetInt("database.maxOpen")
c.dbConfig.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime"))
c.DBStats = viper.GetBool("prom.dbStats")
// Create DB
db, err := shared.NewDB(c.dbConfig.DbConnectionString(), c.dbConfig)
if err != nil {
return fmt.Errorf("failed to create config: %w", err)
}
c.DB = db
return nil
}
func (c *Config) setupEth() error {
var err error
chainConfigPath := viper.GetString("ethereum.chainConfig")
if chainConfigPath != "" {
c.ChainConfig, err = utils.LoadConfig(chainConfigPath)
} else {
// read chainID if chain config path not provided
chainID := viper.GetUint64("ethereum.chainID")
c.ChainConfig, err = utils.ChainConfig(chainID)
}
if err != nil {
return err
}
// setup a statediffing client
ethHTTP := viper.GetString("ethereum.httpPath")
if ethHTTP != "" {
ethHTTPEndpoint := fmt.Sprintf("http://%s", ethHTTP)
c.Client, err = rpc.Dial(ethHTTPEndpoint)
}
return err
}
func (c *Config) setupValidator() error {
var err error
c.FromBlock = viper.GetUint64("validate.fromBlock")
if c.FromBlock < 1 {
return fmt.Errorf("starting block height cannot be less than 1")
}
c.Trail = viper.GetUint64("validate.trail")
c.RetryInterval = viper.GetDuration("validate.retryInterval")
c.StateDiffMissingBlock = viper.GetBool("validate.stateDiffMissingBlock")
if c.StateDiffMissingBlock {
c.StateDiffTimeout = viper.GetDuration("validate.stateDiffTimeout")
}
return err
}

View File

@ -1,19 +1,3 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
import (
@ -25,19 +9,120 @@ import (
var errNotSupported = errors.New("this operation is not supported")
type database struct {
ethdb.Database
ethDB ethdb.Database
}
func newDatabase(db ethdb.Database) *database {
return &database{
Database: db,
ethDB: db,
}
}
func (d *database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
return d.ethDB.NewIterator(prefix, start)
}
func (d *database) Has(key []byte) (bool, error) {
return d.ethDB.Has(key)
}
func (d *database) Get(key []byte) ([]byte, error) {
return d.ethDB.Get(key)
}
func (d *database) Put(key []byte, value []byte) error {
return errNotSupported
return nil
}
func (d *database) Delete(key []byte) error {
return errNotSupported
return nil
}
func (d *database) Stat(property string) (string, error) {
return d.ethDB.Stat(property)
}
func (d *database) Compact(start []byte, limit []byte) error {
return d.ethDB.Compact(start, limit)
}
// HasAncient returns an error as we don't have a backing chain freezer.
func (d *database) HasAncient(kind string, number uint64) (bool, error) {
return d.ethDB.HasAncient(kind, number)
}
// Ancient returns an error as we don't have a backing chain freezer.
func (d *database) Ancient(kind string, number uint64) ([]byte, error) {
return d.ethDB.Ancient(kind, number)
}
// AncientRange returns an error as we don't have a backing chain freezer.
func (d *database) AncientRange(kind string, start, max, maxByteSize uint64) ([][]byte, error) {
return d.ethDB.AncientRange(kind, start, max, maxByteSize)
}
// Ancients returns an error as we don't have a backing chain freezer.
func (d *database) Ancients() (uint64, error) {
return d.ethDB.Ancients()
}
// AncientSize returns an error as we don't have a backing chain freezer.
func (d *database) AncientSize(kind string) (uint64, error) {
return d.ethDB.AncientSize(kind)
}
// Tail returns the number of first stored item in the freezer.
func (d *database) Tail() (uint64, error) {
return d.Tail()
}
// ModifyAncients is not supported.
func (d *database) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (int64, error) {
return 0, nil
}
// TruncateHead discards all but the first n ancient data from the ancient store.
func (d *database) TruncateHead(n uint64) error {
return nil
}
// TruncateTail discards the first n ancient data from the ancient store.
func (d *database) TruncateTail(n uint64) error {
return nil
}
func (d *database) Sync() error {
return d.ethDB.Sync()
}
// MigrateTable processes and migrates entries of a given table to a new format.
func (d *database) MigrateTable(string, func([]byte) ([]byte, error)) error {
return nil
}
func (d *database) NewBatch() ethdb.Batch {
return d.ethDB.NewBatch()
}
// NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
func (d *database) NewBatchWithSize(size int) ethdb.Batch {
return d.ethDB.NewBatchWithSize(size)
}
func (d *database) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
return d.ethDB.ReadAncients(fn)
}
func (d *database) Close() error {
return d.ethDB.Close()
}
// NewSnapshot creates a database snapshot based on the current state.
func (d *database) NewSnapshot() (ethdb.Snapshot, error) {
return d.NewSnapshot()
}
// NewSnapshot creates a database snapshot based on the current state.
func (d *database) AncientDatadir() (string, error) {
return "", errNotSupported
}

View File

@ -1,13 +0,0 @@
package validator
import (
"fmt"
)
type ChainNotSyncedError struct {
Head uint64
}
func (e *ChainNotSyncedError) Error() string {
return fmt.Sprintf("chain not synced (current head: %d)", e.Head)
}

View File

@ -1,19 +1,3 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
import (
@ -22,44 +6,50 @@ import (
"github.com/jmoiron/sqlx"
)
var (
ReferentialIntegrityErr = "referential integrity check failed at block %d, entry for %s not found"
EntryNotFoundErr = "entry for %s not found"
)
// ValidateReferentialIntegrity validates referential integrity at the given height
func ValidateReferentialIntegrity(tx *sqlx.Tx, blockNumber uint64) error {
err := ValidateHeaderCIDsRef(tx, blockNumber)
func ValidateReferentialIntegrity(db *sqlx.DB, blockNumber uint64) error {
err := ValidateHeaderCIDsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateUncleCIDsRef(tx, blockNumber)
err = ValidateUncleCIDsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateTransactionCIDsRef(tx, blockNumber)
err = ValidateTransactionCIDsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateReceiptCIDsRef(tx, blockNumber)
err = ValidateReceiptCIDsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateStateCIDsRef(tx, blockNumber)
err = ValidateStateCIDsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateStorageCIDsRef(tx, blockNumber)
err = ValidateStorageCIDsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateLogCIDsRef(tx, blockNumber)
err = ValidateStateAccountsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateAccessListElementsRef(db, blockNumber)
if err != nil {
return err
}
err = ValidateLogCIDsRef(db, blockNumber)
if err != nil {
return err
}
@ -68,8 +58,8 @@ func ValidateReferentialIntegrity(tx *sqlx.Tx, blockNumber uint64) error {
}
// ValidateHeaderCIDsRef does a reference integrity check on references in eth.header_cids table
func ValidateHeaderCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
err := ValidateIPFSBlocks(tx, blockNumber, "eth.header_cids", "cid")
func ValidateHeaderCIDsRef(db *sqlx.DB, blockNumber uint64) error {
err := ValidateIPFSBlocks(db, blockNumber, "eth.header_cids", "mh_key")
if err != nil {
return err
}
@ -78,9 +68,9 @@ func ValidateHeaderCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
}
// ValidateUncleCIDsRef does a reference integrity check on references in eth.uncle_cids table
func ValidateUncleCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
func ValidateUncleCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := tx.Get(&exists, UncleCIDsRefHeaderCIDs, blockNumber)
err := db.Get(&exists, UncleCIDsRefHeaderCIDs, blockNumber)
if err != nil {
return err
}
@ -88,7 +78,7 @@ func ValidateUncleCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
}
err = ValidateIPFSBlocks(tx, blockNumber, "eth.uncle_cids", "cid")
err = ValidateIPFSBlocks(db, blockNumber, "eth.uncle_cids", "mh_key")
if err != nil {
return err
}
@ -97,9 +87,9 @@ func ValidateUncleCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
}
// ValidateTransactionCIDsRef does a reference integrity check on references in eth.header_cids table
func ValidateTransactionCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
func ValidateTransactionCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := tx.Get(&exists, TransactionCIDsRefHeaderCIDs, blockNumber)
err := db.Get(&exists, TransactionCIDsRefHeaderCIDs, blockNumber)
if err != nil {
return err
}
@ -107,7 +97,7 @@ func ValidateTransactionCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
}
err = ValidateIPFSBlocks(tx, blockNumber, "eth.transaction_cids", "cid")
err = ValidateIPFSBlocks(db, blockNumber, "eth.transaction_cids", "mh_key")
if err != nil {
return err
}
@ -116,9 +106,9 @@ func ValidateTransactionCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
}
// ValidateReceiptCIDsRef does a reference integrity check on references in eth.receipt_cids table
func ValidateReceiptCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
func ValidateReceiptCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := tx.Get(&exists, ReceiptCIDsRefTransactionCIDs, blockNumber)
err := db.Get(&exists, ReceiptCIDsRefTransactionCIDs, blockNumber)
if err != nil {
return err
}
@ -126,7 +116,7 @@ func ValidateReceiptCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
}
err = ValidateIPFSBlocks(tx, blockNumber, "eth.receipt_cids", "cid")
err = ValidateIPFSBlocks(db, blockNumber, "eth.receipt_cids", "leaf_mh_key")
if err != nil {
return err
}
@ -135,9 +125,9 @@ func ValidateReceiptCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
}
// ValidateStateCIDsRef does a reference integrity check on references in eth.state_cids table
func ValidateStateCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
func ValidateStateCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := tx.Get(&exists, StateCIDsRefHeaderCIDs, blockNumber)
err := db.Get(&exists, StateCIDsRefHeaderCIDs, blockNumber)
if err != nil {
return err
}
@ -145,7 +135,7 @@ func ValidateStateCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.header_cids")
}
err = ValidateIPFSBlocks(tx, blockNumber, "eth.state_cids", "cid")
err = ValidateIPFSBlocks(db, blockNumber, "eth.state_cids", "mh_key")
if err != nil {
return err
}
@ -154,9 +144,9 @@ func ValidateStateCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
}
// ValidateStorageCIDsRef does a reference integrity check on references in eth.storage_cids table
func ValidateStorageCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
func ValidateStorageCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := tx.Get(&exists, StorageCIDsRefStateCIDs, blockNumber)
err := db.Get(&exists, StorageCIDsRefStateCIDs, blockNumber)
if err != nil {
return err
}
@ -164,7 +154,7 @@ func ValidateStorageCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
}
err = ValidateIPFSBlocks(tx, blockNumber, "eth.storage_cids", "cid")
err = ValidateIPFSBlocks(db, blockNumber, "eth.storage_cids", "mh_key")
if err != nil {
return err
}
@ -172,10 +162,38 @@ func ValidateStorageCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return nil
}
// ValidateLogCIDsRef does a reference integrity check on references in eth.log_cids table
func ValidateLogCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
// ValidateStateAccountsRef does a reference integrity check on references in eth.state_accounts table
func ValidateStateAccountsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := tx.Get(&exists, LogCIDsRefReceiptCIDs, blockNumber)
err := db.Get(&exists, StateAccountsRefStateCIDs, blockNumber)
if err != nil {
return err
}
if exists {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.state_cids")
}
return nil
}
// ValidateAccessListElementsRef does a reference integrity check on references in eth.access_list_elements table
func ValidateAccessListElementsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := db.Get(&exists, AccessListElementsRefTransactionCIDs, blockNumber)
if err != nil {
return err
}
if exists {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.transaction_cids")
}
return nil
}
// ValidateLogCIDsRef does a reference integrity check on references in eth.log_cids table
func ValidateLogCIDsRef(db *sqlx.DB, blockNumber uint64) error {
var exists bool
err := db.Get(&exists, LogCIDsRefReceiptCIDs, blockNumber)
if err != nil {
return err
}
@ -183,7 +201,7 @@ func ValidateLogCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "eth.receipt_cids")
}
err = ValidateIPFSBlocks(tx, blockNumber, "eth.log_cids", "cid")
err = ValidateIPFSBlocks(db, blockNumber, "eth.log_cids", "leaf_mh_key")
if err != nil {
return err
}
@ -192,14 +210,14 @@ func ValidateLogCIDsRef(tx *sqlx.Tx, blockNumber uint64) error {
}
// ValidateIPFSBlocks does a reference integrity check between the given CID table and IPFS blocks table on MHKey and block number
func ValidateIPFSBlocks(tx *sqlx.Tx, blockNumber uint64, CIDTable string, CIDField string) error {
func ValidateIPFSBlocks(db *sqlx.DB, blockNumber uint64, CIDTable string, mhKeyField string) error {
var exists bool
err := tx.Get(&exists, fmt.Sprintf(CIDsRefIPLDBlocks, CIDTable, CIDField), blockNumber)
err := db.Get(&exists, fmt.Sprintf(CIDsRefIPLDBlocks, CIDTable, mhKeyField), blockNumber)
if err != nil {
return err
}
if exists {
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "ipld.blocks")
return fmt.Errorf(ReferentialIntegrityErr, blockNumber, "public.blocks")
}
return nil

View File

@ -1,19 +1,3 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
// Queries to validate referential integrity in the indexed data:
@ -26,7 +10,7 @@ const (
CIDsRefIPLDBlocks = `SELECT EXISTS (
SELECT *
FROM %[1]s
LEFT JOIN ipld.blocks ON (
LEFT JOIN public.blocks ON (
%[1]s.%[2]s = blocks.key
AND %[1]s.block_number = blocks.block_number
)
@ -88,13 +72,38 @@ const (
SELECT *
FROM eth.storage_cids
LEFT JOIN eth.state_cids ON (
storage_cids.state_leaf_key = state_cids.state_leaf_key
storage_cids.state_path = state_cids.state_path
AND storage_cids.header_id = state_cids.header_id
AND storage_cids.block_number = state_cids.block_number
)
WHERE
storage_cids.block_number = $1
AND state_cids.state_leaf_key IS NULL
AND state_cids.state_path IS NULL
)`
StateAccountsRefStateCIDs = `SELECT EXISTS (
SELECT *
FROM eth.state_accounts
LEFT JOIN eth.state_cids ON (
state_accounts.state_path = state_cids.state_path
AND state_accounts.header_id = state_cids.header_id
AND state_accounts.block_number = state_cids.block_number
)
WHERE
state_accounts.block_number = $1
AND state_cids.state_path IS NULL
)`
AccessListElementsRefTransactionCIDs = `SELECT EXISTS (
SELECT *
FROM eth.access_list_elements
LEFT JOIN eth.transaction_cids ON (
access_list_elements.tx_id = transaction_cids.tx_hash
AND access_list_elements.block_number = transaction_cids.block_number
)
WHERE
access_list_elements.block_number = $1
AND transaction_cids.tx_hash IS NULL
)`
LogCIDsRefReceiptCIDs = `SELECT EXISTS (

View File

@ -3,256 +3,332 @@ package validator_test
import (
"context"
"fmt"
"math/big"
"testing"
indexer_helpers "github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
helpers "github.com/cerc-io/plugeth-statediff/test_helpers"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/jmoiron/sqlx"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/cerc-io/ipld-eth-db-validator/v5/internal/chaingen"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/validator"
"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
"github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
)
func TestRefIntegrity(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "ETH IPLD validator ref integrity suite test")
}
var _ = Describe("referential integrity", Ordered, func() {
var _ = Describe("RefIntegrity", func() {
var (
ctx = context.Background()
db *sqlx.DB
tx *sqlx.Tx
checkedBlock *types.Block // Generated block of interest
)
BeforeAll(func() {
var (
blocks []*types.Block
receipts []types.Receipts
chain *core.BlockChain
chainConfig = TestChainConfig
mockTD = big.NewInt(1337)
testdb = rawdb.NewMemoryDatabase()
diffIndexer interfaces.StateDiffIndexer
)
gen := chaingen.DefaultGenContext(chainConfig, testdb)
gen.AddFunction(func(i int, block *core.BlockGen) {
if i >= 2 {
uncle := &types.Header{
Number: big.NewInt(int64(i - 1)),
Root: common.HexToHash("0x1"),
TxHash: common.HexToHash("0x1"),
ReceiptHash: common.HexToHash("0x1"),
ParentHash: block.PrevBlock(i - 1).Hash(),
}
block.AddUncle(uncle)
}
BeforeEach(func() {
db = shared.SetupDB()
diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
})
blocks, receipts, chain = gen.MakeChain(5)
indexer, err := helpers.NewIndexer(context.Background(), chainConfig, gen.Genesis.Hash(), TestDBConfig)
Expect(err).ToNot(HaveOccurred())
helpers.IndexChain(indexer, helpers.IndexChainParams{
StateCache: chain.StateCache(),
Blocks: blocks,
Receipts: receipts,
TotalDifficulty: mockTD,
})
checkedBlock = blocks[5]
db = SetupDB()
})
AfterAll(func() { Expect(indexer_helpers.ClearSqlxDB(db)).ToNot(HaveOccurred()) })
BeforeEach(func() { tx = db.MustBegin() })
AfterEach(func() {
tx.Rollback()
shared.TearDownDB(db)
})
Describe("ValidateHeaderCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of header_cids table", func() {
err := validator.ValidateHeaderCIDsRef(tx, checkedBlock.NumberU64())
err := validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding header IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks")
err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateHeaderCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateHeaderCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks"))
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
})
})
Describe("ValidateUncleCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of uncle_cids table", func() {
err := validator.ValidateUncleCIDsRef(tx, checkedBlock.NumberU64())
err := validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding header_cid entry not found", func() {
err := deleteEntriesFrom(tx, "eth.header_cids")
err := deleteEntriesFrom(db, "eth.header_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateUncleCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
})
It("Throws an error if corresponding uncle IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks")
err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateUncleCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateUncleCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks"))
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
})
})
Describe("ValidateTransactionCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of transaction_cids table", func() {
err := validator.ValidateTransactionCIDsRef(tx, checkedBlock.NumberU64())
err := validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding header_cid entry not found", func() {
err := deleteEntriesFrom(tx, "eth.header_cids")
err := deleteEntriesFrom(db, "eth.header_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateTransactionCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
})
It("Throws an error if corresponding transaction IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks")
err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateTransactionCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateTransactionCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks"))
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
})
})
Describe("ValidateReceiptCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of receipt_cids table", func() {
err := validator.ValidateReceiptCIDsRef(tx, checkedBlock.NumberU64())
err := validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding transaction_cids entry not found", func() {
err := deleteEntriesFrom(tx, "eth.transaction_cids")
err := deleteEntriesFrom(db, "eth.transaction_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateReceiptCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
})
It("Throws an error if corresponding receipt IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks")
err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateReceiptCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateReceiptCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks"))
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
})
})
Describe("ValidateStateCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of state_cids table", func() {
err := validator.ValidateStateCIDsRef(tx, checkedBlock.NumberU64())
err := validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding header_cids entry not found", func() {
err := deleteEntriesFrom(tx, "eth.header_cids")
err := deleteEntriesFrom(db, "eth.header_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStateCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.header_cids"))
})
It("Throws an error if corresponding state IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks")
err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStateCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateStateCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks"))
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
})
})
Describe("ValidateStorageCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of storage_cids table", func() {
err := validator.ValidateStorageCIDsRef(tx, checkedBlock.NumberU64())
err := validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding state_cids entry not found", func() {
err := deleteEntriesFrom(tx, "eth.state_cids")
err := deleteEntriesFrom(db, "eth.state_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStorageCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
})
It("Throws an error if corresponding storage IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks")
err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStorageCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateStorageCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks"))
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
})
})
Describe("ValidateStateAccountsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of state_accounts table", func() {
err := validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding state_cids entry not found", func() {
err := deleteEntriesFrom(db, "eth.state_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateStateAccountsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.state_cids"))
})
})
Describe("ValidateAccessListElementsRef", func() {
BeforeEach(func() {
indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, mocks.TestConfig, test_helpers.Genesis.Hash())
tx, err := indexAndPublisher.PushBlock(mocks.MockBlock, mocks.MockReceipts, mocks.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of access_list_elements table", func() {
err := validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding transaction_cids entry not found", func() {
err := deleteEntriesFrom(db, "eth.transaction_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateAccessListElementsRef(db, mocks.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.transaction_cids"))
})
})
Describe("ValidateLogCIDsRef", func() {
BeforeEach(func() {
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
It("Validates referential integrity of log_cids table", func() {
err := validator.ValidateLogCIDsRef(tx, checkedBlock.NumberU64())
err := validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).ToNot(HaveOccurred())
})
It("Throws an error if corresponding receipt_cids entry not found", func() {
err := deleteEntriesFrom(tx, "eth.receipt_cids")
err := deleteEntriesFrom(db, "eth.receipt_cids")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateLogCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "eth.receipt_cids"))
})
It("Throws an error if corresponding log IPFS block entry not found", func() {
err := deleteEntriesFrom(tx, "ipld.blocks")
err := deleteEntriesFrom(db, "public.blocks")
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateLogCIDsRef(tx, checkedBlock.NumberU64())
err = validator.ValidateLogCIDsRef(db, test_helpers.MockBlock.NumberU64())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "ipld.blocks"))
Expect(err.Error()).To(ContainSubstring(validator.EntryNotFoundErr, "public.blocks"))
})
})
Describe("ValidateReferentialIntegrity", func() {
It("Validates referential integrity of full chain", func() {
for i := uint64(startBlock); i <= chainLength; i++ {
err := validator.ValidateReferentialIntegrity(tx, i)
Expect(err).ToNot(HaveOccurred())
}
})
})
})
func deleteEntriesFrom(tx *sqlx.Tx, tableName string) error {
pgStr := "TRUNCATE %s"
_, err := tx.Exec(fmt.Sprintf(pgStr, tableName))
func deleteEntriesFrom(db *sqlx.DB, tableName string) error {
pgStr := "DELETE FROM %s"
_, err := db.Exec(fmt.Sprintf(pgStr, tableName))
return err
}

View File

@ -1,46 +0,0 @@
package validator_test
import (
"context"
"math/big"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/params"
"github.com/jmoiron/sqlx"
)
var TestChainConfig = &params.ChainConfig{
ChainID: big.NewInt(1),
HomesteadBlock: big.NewInt(0),
EIP150Block: big.NewInt(0),
EIP155Block: big.NewInt(0),
EIP158Block: big.NewInt(0),
ByzantiumBlock: big.NewInt(0),
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
MuirGlacierBlock: big.NewInt(0),
BerlinBlock: big.NewInt(0),
LondonBlock: big.NewInt(6),
ArrowGlacierBlock: big.NewInt(0),
GrayGlacierBlock: big.NewInt(0),
Ethash: new(params.EthashConfig),
}
var TestDBConfig postgres.Config
func init() {
var err error
TestDBConfig, err = postgres.TestConfig.WithEnv()
if err != nil {
panic(err)
}
}
func SetupDB() *sqlx.DB {
db, err := postgres.ConnectSQLX(context.Background(), TestDBConfig)
if err != nil {
panic(err)
}
return db
}

View File

@ -1,211 +1,193 @@
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
import (
"context"
"encoding/json"
"errors"
"fmt"
"math/big"
"sync"
"time"
statediff "github.com/cerc-io/plugeth-statediff"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/clique"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
"github.com/holiman/uint256"
"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
ipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0"
ipldeth "github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
ipldstate "github.com/cerc-io/ipld-eth-statedb/trie_by_cid/state"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/prom"
ipfsethdb "github.com/vulcanize/ipfs-ethdb/v4/postgres"
ipldEth "github.com/vulcanize/ipld-eth-server/v4/pkg/eth"
ethServerShared "github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
)
var (
big8 = big.NewInt(8)
big32 = big.NewInt(32)
ReferentialIntegrityErr = "referential integrity check failed at block %d, entry for %s not found"
EntryNotFoundErr = "entry for %s not found"
)
type Service struct {
type service struct {
db *sqlx.DB
chainConfig *params.ChainConfig
ethClient *rpc.Client
blockNum, trail uint64
retryInterval time.Duration
stateDiffMissingBlock bool
stateDiffTimeout time.Duration
sleepInterval uint
logger *log.Logger
chainCfg *params.ChainConfig
quitChan chan bool
progressChan chan<- uint64
progressChan chan uint64
}
func NewService(cfg *Config, progressChan chan<- uint64) (*Service, error) {
db, err := postgres.ConnectSQLX(context.Background(), cfg.DBConfig)
if err != nil {
return nil, err
}
// Enable DB stats
if cfg.DBStats {
prom.RegisterDBCollector(cfg.DBConfig.DatabaseName, db)
}
return &Service{
db: db,
chainConfig: cfg.ChainConfig,
ethClient: cfg.Client,
blockNum: cfg.FromBlock,
func NewService(cfg *Config, progressChan chan uint64) *service {
return &service{
db: cfg.DB,
blockNum: cfg.BlockNum,
trail: cfg.Trail,
retryInterval: cfg.RetryInterval,
stateDiffMissingBlock: cfg.StateDiffMissingBlock,
stateDiffTimeout: cfg.StateDiffTimeout,
sleepInterval: cfg.SleepInterval,
logger: log.New(),
chainCfg: cfg.ChainCfg,
quitChan: make(chan bool),
progressChan: progressChan,
}
}
func NewEthBackend(db *sqlx.DB, c *ipldEth.Config) (*ipldEth.Backend, error) {
gcc := c.GroupCacheConfig
groupName := gcc.StateDB.Name
if groupName == "" {
groupName = ipldEth.StateDBGroupCacheName
}
r := ipldEth.NewCIDRetriever(db)
ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{
Name: groupName,
Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024,
ExpiryDuration: time.Minute * time.Duration(gcc.StateDB.CacheExpiryInMins),
})
// Read only wrapper around ipfs-ethdb eth.Database implementation
customEthDB := newDatabase(ethDB)
return &ipldEth.Backend{
DB: db,
Retriever: r,
Fetcher: ipldEth.NewIPLDFetcher(db),
IPLDRetriever: ipldEth.NewIPLDRetriever(db),
EthDB: customEthDB,
StateDatabase: state.NewDatabase(customEthDB),
Config: c,
}, nil
}
// Start is used to begin the service
func (s *Service) Start(ctx context.Context, wg *sync.WaitGroup) {
func (s *service) Start(ctx context.Context, wg *sync.WaitGroup) {
defer wg.Done()
api, err := EthAPI(ctx, s.db, s.chainConfig)
api, err := EthAPI(ctx, s.db, s.chainCfg)
if err != nil {
log.Fatal(err)
s.logger.Fatal(err)
return
}
nextBlockNum := s.blockNum
var delay time.Duration
idxBlockNum := s.blockNum
for {
select {
case <-s.quitChan:
log.Info("stopping ipld-eth-db-validator process")
s.logger.Infof("last validated block %v", idxBlockNum-1)
if s.progressChan != nil {
close(s.progressChan)
}
if err := api.B.Close(); err != nil {
log.Errorf("error closing backend: %s", err)
}
return
case <-time.After(delay):
err := s.Validate(ctx, api, nextBlockNum)
// If chain is not synced, wait for trail to catch up before trying again
if notsynced, ok := err.(*ChainNotSyncedError); ok {
delay = s.retryInterval
log.Infof("waiting %v for chain to advance to block %d (head is at %d)",
delay, nextBlockNum+s.trail, notsynced.Head)
continue
}
default:
idxBlockNum, err = s.Validate(ctx, api, idxBlockNum)
if err != nil {
log.Fatal(err)
s.logger.Infof("last validated block %v", idxBlockNum-1)
s.logger.Fatal(err)
return
}
prom.SetLastValidatedBlock(float64(nextBlockNum))
nextBlockNum++
delay = 0
}
}
}
// Stop is used to gracefully stop the service
func (s *Service) Stop() {
func (s *service) Stop() {
s.logger.Info("stopping ipld-eth-db-validator process")
close(s.quitChan)
}
func (s *Service) Validate(ctx context.Context, api *ipldeth.PublicEthAPI, idxBlockNum uint64) error {
log.Debugf("validating block %d", idxBlockNum)
func (s *service) Validate(ctx context.Context, api *ipldEth.PublicEthAPI, idxBlockNum uint64) (uint64, error) {
headBlockNum, err := fetchHeadBlockNumber(ctx, api)
if err != nil {
return err
return idxBlockNum, err
}
// Check if block at requested height can be validated
if idxBlockNum+s.trail > headBlockNum {
return &ChainNotSyncedError{headBlockNum}
}
blockToBeValidated, err := api.B.BlockByNumber(ctx, rpc.BlockNumber(idxBlockNum))
// Check if it block at height idxBlockNum can be validated
if idxBlockNum <= headBlockNum-s.trail {
err = ValidateBlock(ctx, api, idxBlockNum)
if err != nil {
log.Errorf("failed to fetch block at height %d", idxBlockNum)
return err
s.logger.Errorf("failed to verify state root at block %d", idxBlockNum)
return idxBlockNum, err
}
// Make a writeStateDiffAt call if block not found in the db
if blockToBeValidated == nil {
return s.writeStateDiffAt(idxBlockNum)
}
s.logger.Infof("state root verified for block %d", idxBlockNum)
err = ValidateBlock(blockToBeValidated, api.B, idxBlockNum)
err = ValidateReferentialIntegrity(s.db, idxBlockNum)
if err != nil {
log.Errorf("failed to verify state root at block %d", idxBlockNum)
return err
s.logger.Errorf("failed to verify referential integrity at block %d", idxBlockNum)
return idxBlockNum, err
}
log.Infof("state root verified for block %d", idxBlockNum)
tx := s.db.MustBegin()
defer tx.Rollback()
err = ValidateReferentialIntegrity(tx, idxBlockNum)
if err != nil {
log.Errorf("failed to verify referential integrity at block %d", idxBlockNum)
return err
}
log.Infof("referential integrity verified for block %d", idxBlockNum)
s.logger.Infof("referential integrity verified for block %d", idxBlockNum)
if s.progressChan != nil {
s.progressChan <- idxBlockNum
}
return nil
idxBlockNum++
} else {
// Sleep / wait for head to move ahead
time.Sleep(time.Second * time.Duration(s.sleepInterval))
}
return idxBlockNum, nil
}
// ValidateBlock validates block at the given height
func ValidateBlock(blockToBeValidated *types.Block, b *ipldeth.Backend, blockNumber uint64) error {
state, err := applyTransactions(blockToBeValidated, b)
func ValidateBlock(ctx context.Context, api *ipldEth.PublicEthAPI, blockNumber uint64) error {
blockToBeValidated, err := api.B.BlockByNumber(ctx, rpc.BlockNumber(blockNumber))
if err != nil {
return err
}
blockStateRoot := blockToBeValidated.Header().Root
dbStateRoot := state.IntermediateRoot(true)
stateDB, err := applyTransaction(blockToBeValidated, api.B)
if err != nil {
return err
}
blockStateRoot := blockToBeValidated.Header().Root.String()
dbStateRoot := stateDB.IntermediateRoot(true).String()
if blockStateRoot != dbStateRoot {
return fmt.Errorf("state roots do not match at block %d", blockNumber)
}
return nil
}
func EthAPI(ctx context.Context, db *sqlx.DB, chainCfg *params.ChainConfig) (*ipldeth.PublicEthAPI, error) {
func EthAPI(ctx context.Context, db *sqlx.DB, chainCfg *params.ChainConfig) (*ipldEth.PublicEthAPI, error) {
// TODO: decide network for custom chainConfig.
backend, err := ethBackend(db, &ipldeth.Config{
backend, err := NewEthBackend(db, &ipldEth.Config{
ChainConfig: chainCfg,
GroupCacheConfig: &shared.GroupCacheConfig{
StateDB: shared.GroupConfig{
Name: "cerc_validator",
GroupCacheConfig: &ethServerShared.GroupCacheConfig{
StateDB: ethServerShared.GroupConfig{
Name: "vulcanize_validator",
},
},
})
@ -222,44 +204,12 @@ func EthAPI(ctx context.Context, db *sqlx.DB, chainCfg *params.ChainConfig) (*ip
}
backend.Config.ChainConfig = setChainConfig(genesisBlock.Hash())
}
config := ipldeth.APIConfig{
SupportsStateDiff: false,
ForwardEthCalls: false,
ForwardGetStorageAt: false,
ProxyOnError: false,
StateDiffTimeout: 0,
}
return ipldeth.NewPublicEthAPI(backend, nil, config)
}
func ethBackend(db *sqlx.DB, c *ipldeth.Config) (*ipldeth.Backend, error) {
gcc := c.GroupCacheConfig
groupName := gcc.StateDB.Name
if groupName == "" {
groupName = ipldeth.StateDBGroupCacheName
}
r := ipldeth.NewRetriever(db)
ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{
Name: groupName,
Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024,
ExpiryDuration: time.Minute * time.Duration(gcc.StateDB.CacheExpiryInMins),
})
// Read only wrapper around ipfs-ethdb eth.Database implementation
ethDB = newDatabase(ethDB)
return &ipldeth.Backend{
DB: db,
Retriever: r,
EthDB: ethDB,
IpldTrieStateDatabase: ipldstate.NewDatabase(ethDB),
Config: c,
}, nil
return ipldEth.NewPublicEthAPI(backend, nil, false, false, false)
}
// fetchHeadBlockNumber gets the latest block number from the db
func fetchHeadBlockNumber(ctx context.Context, api *ipldeth.PublicEthAPI) (uint64, error) {
func fetchHeadBlockNumber(ctx context.Context, api *ipldEth.PublicEthAPI) (uint64, error) {
headBlock, err := api.B.BlockByNumber(ctx, rpc.LatestBlockNumber)
if err != nil {
return 0, err
@ -268,101 +218,58 @@ func fetchHeadBlockNumber(ctx context.Context, api *ipldeth.PublicEthAPI) (uint6
return headBlock.NumberU64(), nil
}
// writeStateDiffAt calls out to a statediffing geth client to fill in a gap in the index
func (s *Service) writeStateDiffAt(height uint64) error {
if !s.stateDiffMissingBlock {
return nil
}
var data json.RawMessage
params := statediff.Params{
IncludeBlock: true,
IncludeReceipts: true,
IncludeTD: true,
IncludeCode: true,
}
ctx, cancel := context.WithTimeout(context.Background(), s.stateDiffTimeout)
defer cancel()
log.Warnf("calling writeStateDiffAt at block %d", height)
if err := s.ethClient.CallContext(ctx, &data, "statediff_writeStateDiffAt", height, params); err != nil {
log.Errorf("writeStateDiffAt %d failed with err %s", height, err)
return err
}
return nil
}
// applyTransaction attempts to apply block transactions to the given state database
// and uses the input parameters for its environment. It returns the stateDB of parent with applied txs.
//
// Note: this skips the DAO hard fork refund
func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate.StateDB, error) {
// applyTransaction attempts to apply a transaction to the given state database
// and uses the input parameters for its environment. It returns the stateDB of parent with applied transa
func applyTransaction(block *types.Block, backend *ipldEth.Backend) (*state.StateDB, error) {
if block.NumberU64() == 0 {
return nil, errors.New("no transaction in genesis")
}
config := backend.Config.ChainConfig
// Create the parent state database
parentHash := block.ParentHash()
nrOrHash := rpc.BlockNumberOrHash{BlockHash: &parentHash}
statedb, _, err := backend.IPLDTrieStateDBAndHeaderByNumberOrHash(context.Background(), nrOrHash)
parentRPCBlockHash := rpc.BlockNumberOrHash{
BlockHash: &parentHash,
RequireCanonical: false,
}
parent, _ := backend.BlockByNumberOrHash(context.Background(), parentRPCBlockHash)
if parent == nil {
return nil, fmt.Errorf("parent %#x not found", block.ParentHash())
}
stateDB, _, err := backend.StateAndHeaderByNumberOrHash(context.Background(), parentRPCBlockHash)
if err != nil {
return nil, fmt.Errorf("error accessing state DB: %w", err)
return nil, err
}
var gp core.GasPool
gp.AddGas(block.GasLimit())
signer := types.MakeSigner(backend.Config.ChainConfig, block.Number())
blockContext := core.NewEVMBlockContext(block.Header(), backend, getAuthor(backend, block.Header()))
evm := vm.NewEVM(blockContext, vm.TxContext{}, statedb, config, vm.Config{})
signer := types.MakeSigner(config, block.Number(), block.Time())
for idx, tx := range block.Transactions() {
// Assemble the transaction call message and return if the requested offset
msg, _ := tx.AsMessage(signer, block.BaseFee())
txContext := core.NewEVMTxContext(msg)
ctx := core.NewEVMBlockContext(block.Header(), backend, getAuthor(backend, block.Header()))
if beaconRoot := block.BeaconRoot(); beaconRoot != nil {
ProcessBeaconBlockRoot(*beaconRoot, evm, statedb)
}
// Not yet the searched for transaction, execute on top of the current state
newEVM := vm.NewEVM(ctx, txContext, stateDB, backend.Config.ChainConfig, vm.Config{})
// Iterate over and process the individual transactions
for i, tx := range block.Transactions() {
msg, err := core.TransactionToMessage(tx, signer, block.BaseFee())
if err != nil {
return nil, fmt.Errorf("error converting transaction to message: %w", err)
}
statedb.SetTxContext(tx.Hash(), i)
// Create a new context to be used in the EVM environment.
evm.Reset(core.NewEVMTxContext(msg), statedb)
// Apply the transaction to the current state (included in the env).
if _, err := core.ApplyMessage(evm, msg, &gp); err != nil {
return nil, fmt.Errorf("error applying tx %#x: %w", tx.Hash(), err)
}
if config.IsByzantium(block.Number()) {
statedb.Finalise(true)
} else {
statedb.IntermediateRoot(config.IsEIP158(block.Number())).Bytes()
stateDB.Prepare(tx.Hash(), idx)
if _, err := core.ApplyMessage(newEVM, msg, new(core.GasPool).AddGas(block.GasLimit())); err != nil {
return nil, fmt.Errorf("transaction %#x failed: %w", tx.Hash(), err)
}
}
// Withdrawals processing.
for _, w := range block.Withdrawals() {
// Convert amount from gwei to wei.
amount := new(uint256.Int).SetUint64(w.Amount)
amount = amount.Mul(amount, uint256.NewInt(params.GWei))
statedb.AddBalance(w.Address, amount)
if backend.Config.ChainConfig.Ethash != nil {
accumulateRewards(backend.Config.ChainConfig, stateDB, block.Header(), block.Uncles())
}
if config.Ethash != nil {
accumulateRewards(config, statedb, block.Header(), block.Uncles())
}
return statedb, nil
return stateDB, nil
}
// accumulateRewards credits the coinbase of the given block with the mining
// reward. The total reward consists of the static block reward and rewards for
// included uncles. The coinbase of each uncle block is also rewarded.
func accumulateRewards(config *params.ChainConfig, state *ipldstate.StateDB, header *types.Header, uncles []*types.Header) {
func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) {
// Select the correct block reward based on chain progression
blockReward := ethash.FrontierBlockReward
if config.IsByzantium(header.Number) {
@ -374,48 +281,32 @@ func accumulateRewards(config *params.ChainConfig, state *ipldstate.StateDB, hea
}
// Accumulate the rewards for the miner and any included uncles
reward := new(big.Int).Set(blockReward.ToBig())
reward := new(big.Int).Set(blockReward)
r := new(big.Int)
for _, uncle := range uncles {
r.Add(uncle.Number, big8)
r.Sub(r, header.Number)
r.Mul(r, blockReward.ToBig())
r.Mul(r, blockReward)
r.Div(r, big8)
state.AddBalance(uncle.Coinbase, uint256.MustFromBig(r))
state.AddBalance(uncle.Coinbase, r)
r.Div(blockReward.ToBig(), big32)
r.Div(blockReward, big32)
reward.Add(reward, r)
}
state.AddBalance(header.Coinbase, uint256.MustFromBig(reward))
}
// ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root
// contract. This method is exported to be used in tests.
func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb *ipldstate.StateDB) {
// If EIP-4788 is enabled, we need to invoke the beaconroot storage contract with
// the new root
msg := &core.Message{
From: params.SystemAddress,
GasLimit: 30_000_000,
GasPrice: common.Big0,
GasFeeCap: common.Big0,
GasTipCap: common.Big0,
To: &params.BeaconRootsStorageAddress,
Data: beaconRoot[:],
}
vmenv.Reset(core.NewEVMTxContext(msg), statedb)
statedb.AddAddressToAccessList(params.BeaconRootsStorageAddress)
_, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560)
statedb.Finalise(true)
state.AddBalance(header.Coinbase, reward)
}
func setChainConfig(ghash common.Hash) *params.ChainConfig {
switch {
case ghash == params.MainnetGenesisHash:
return params.MainnetChainConfig
case ghash == params.RopstenGenesisHash:
return params.RopstenChainConfig
case ghash == params.SepoliaGenesisHash:
return params.SepoliaChainConfig
case ghash == params.RinkebyGenesisHash:
return params.RinkebyChainConfig
case ghash == params.GoerliGenesisHash:
return params.GoerliChainConfig
default:
@ -423,7 +314,7 @@ func setChainConfig(ghash common.Hash) *params.ChainConfig {
}
}
func getAuthor(b *ipldeth.Backend, header *types.Header) *common.Address {
func getAuthor(b *ipldEth.Backend, header *types.Header) *common.Address {
author, err := getEngine(b).Author(header)
if err != nil {
return nil
@ -432,7 +323,7 @@ func getAuthor(b *ipldeth.Backend, header *types.Header) *common.Address {
return &author
}
func getEngine(b *ipldeth.Backend) consensus.Engine {
func getEngine(b *ipldEth.Backend) consensus.Engine {
// TODO: add logic for other engines
if b.Config.ChainConfig.Clique != nil {
engine := clique.New(b.Config.ChainConfig.Clique, nil)

View File

@ -0,0 +1,13 @@
package validator_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestValidator(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Validator Suite")
}

View File

@ -1,134 +0,0 @@
package validator_test
import (
"context"
"math/big"
"testing"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
indexer_helpers "github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
helpers "github.com/cerc-io/plugeth-statediff/test_helpers"
sdtypes "github.com/cerc-io/plugeth-statediff/types"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
"github.com/jmoiron/sqlx"
// import server helpers for non-canonical chain data
server_mocks "github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
"github.com/cerc-io/ipld-eth-db-validator/v5/internal/chaingen"
"github.com/cerc-io/ipld-eth-db-validator/v5/pkg/validator"
)
const (
chainLength = 10
startBlock = 1
)
var (
chainConfig = TestChainConfig
mockTD = big.NewInt(1337)
testDB = rawdb.NewMemoryDatabase()
)
func init() {
// The geth sync logs are noisy, silence them
log.SetDefault(log.NewLogger(log.DiscardHandler()))
}
func setupStateValidator(t *testing.T) *sqlx.DB {
// Make the test blockchain and state
gen := chaingen.DefaultGenContext(chainConfig, testDB)
blocks, receipts, chain := gen.MakeChain(chainLength)
t.Cleanup(func() {
chain.Stop()
})
indexer, err := helpers.NewIndexer(context.Background(), chainConfig, gen.Genesis.Hash(), TestDBConfig)
if err != nil {
t.Fatal(err)
}
// Insert some non-canonical data into the database so that we test our ability to discern canonicity
// Use a func here for defer
func() {
tx, err := indexer.PushBlock(server_mocks.MockBlock, server_mocks.MockReceipts,
server_mocks.MockBlock.Difficulty())
if err != nil {
t.Fatal(err)
}
defer tx.RollbackOnFailure(err)
err = tx.Submit()
if err != nil {
t.Fatal(err)
}
// The non-canonical header has a child
tx, err = indexer.PushBlock(server_mocks.MockChild, server_mocks.MockReceipts,
server_mocks.MockChild.Difficulty())
if err != nil {
t.Fatal(err)
}
defer tx.RollbackOnFailure(err)
ipld := sdtypes.IPLD{
CID: ipld.Keccak256ToCid(ipld.RawBinary, server_mocks.CodeHash.Bytes()).String(),
Content: server_mocks.ContractCode,
}
err = indexer.PushIPLD(tx, ipld)
if err != nil {
t.Fatal(err)
}
err = tx.Submit()
if err != nil {
t.Fatal(err)
}
}()
if err := helpers.IndexChain(indexer, helpers.IndexChainParams{
StateCache: chain.StateCache(),
Blocks: blocks,
Receipts: receipts,
TotalDifficulty: mockTD,
}); err != nil {
t.Fatal(err)
}
db := SetupDB()
t.Cleanup(func() {
if err := indexer_helpers.ClearSqlxDB(db); err != nil {
t.Fatal(err)
}
})
return db
}
func TestStateValidation(t *testing.T) {
db := setupStateValidator(t)
t.Run("Validator", func(t *testing.T) {
api, err := validator.EthAPI(context.Background(), db, chainConfig)
if err != nil {
t.Fatal(err)
}
for i := uint64(startBlock); i <= chainLength; i++ {
blockToBeValidated, err := api.B.BlockByNumber(context.Background(), rpc.BlockNumber(i))
if err != nil {
t.Fatal(err)
}
if blockToBeValidated == nil {
t.Fatal("block was not found")
}
err = validator.ValidateBlock(blockToBeValidated, api.B, i)
if err != nil {
t.Fatal(err)
}
}
})
}

View File

@ -1,12 +0,0 @@
#!/bin/bash
set -eu
geth_endpoint="${1:-$ETH_HTTP_PATH}"
latest_block_hex=$(curl -s $geth_endpoint -X POST -H "Content-Type: application/json" \
--data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":42}' | \
python3 -c 'import json, sys; print(int(json.load(sys.stdin)["result"], 16))' \
)
printf "%d" $latest_block_hex

View File

@ -1,55 +0,0 @@
#!/bin/bash
set -ex -o pipefail
stack_dir=$(readlink -f "$1")
[[ -d "$stack_dir" ]]
CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
# By default assume we are running in the project root.
export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}"
laconic_so="laconic-so --verbose --stack $stack_dir"
# Don't run geth/plugeth in the debugger, it will swallow error backtraces
echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
# Passing this lets us run eth_call forwarding tests without running ipld-eth-db
echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env
if [[ -z $SKIP_BUILD ]]; then
# Prevent conflicting tty output
export BUILDKIT_PROGRESS=plain
$laconic_so setup-repositories \
--exclude git.vdb.to/cerc-io/ipld-eth-server
$laconic_so build-containers \
--exclude cerc/ipld-eth-server
fi
if ! $laconic_so deploy \
--exclude ipld-eth-server \
--env-file $CONFIG_DIR/stack.env \
--cluster test up
then
$laconic_so deploy --cluster test logs
exit 1
fi
set +x
# Get IPv4 endpoint of geth and bootnode file server
bootnode_endpoint=localhost:$(docker port test-fixturenet-eth-bootnode-geth-1 9898 | head -1 | cut -d':' -f2)
geth_endpoint=localhost:$(docker port test-fixturenet-eth-geth-1-1 8545 | head -1 | cut -d':' -f2)
# Extract the chain config and ID from genesis file
curl -s $bootnode_endpoint/geth.json | jq '.config' > "$CONFIG_DIR/chain.json"
# Output vars if we are running on Github
if [[ -n "$GITHUB_ENV" ]]; then
echo ETH_CHAIN_ID="$(jq '.chainId' $CONFIG_DIR/chain.json)" >> "$GITHUB_ENV"
echo ETH_CHAIN_CONFIG="$CONFIG_DIR/chain.json" >> "$GITHUB_ENV"
echo ETH_HTTP_PATH=$geth_endpoint >> "$GITHUB_ENV"
# Read a private key so we can send from a funded account
echo DEPLOYER_PRIVATE_KEY="$(curl -s $bootnode_endpoint/accounts.csv | head -1 | cut -d',' -f3)" >> "$GITHUB_ENV"
fi

15
scripts/run_integration_test.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash
set -e
set -o xtrace
export PGPASSWORD=password
export DATABASE_USER=vdbm
export DATABASE_PORT=8077
export DATABASE_PASSWORD=password
export DATABASE_HOSTNAME=127.0.0.1
export DATABASE_NAME=vulcanize_testing
# Wait for containers to be up and execute the integration test.
while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
make integrationtest

17
scripts/run_unit_test.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
set -e
# Clear up existing docker images and volume.
docker-compose down --remove-orphans --volumes
# Spin up TimescaleDB
docker-compose -f docker-compose.yml up -d migrations ipld-eth-db
sleep 45
# Run unit tests
go clean -testcache
PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test
# Clean up
docker-compose down --remove-orphans --volumes

91
test/README.md Normal file
View File

@ -0,0 +1,91 @@
# Test Instructions
## Setup
- For running integration tests:
- Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator), [go-ethereum](https://github.com/vulcanize/go-ethereum) and [ipld-eth-db](https://github.com/vulcanize/ipld-eth-db) repositories.
- Checkout [v4 release](https://github.com/vulcanize/ipld-eth-db/releases/tag/v4.2.0-alpha) in ipld-eth-db repo.
```bash
# In ipld-eth-db repo.
git checkout v4.2.0-alpha
```
- Checkout [v4 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.19-statediff-4.1.0-alpha) in go-ethereum repo.
```bash
# In go-ethereum repo.
git checkout v1.10.19-statediff-4.1.0-alpha
```
- Checkout working commit in stack-orchestrator repo.
```bash
# In stack-orchestrator repo.
git checkout f2fd766f5400fcb9eb47b50675d2e3b1f2753702
```
## Run
- Run unit tests:
```bash
# In ipld-eth-db-validator root directory.
./scripts/run_unit_test.sh
```
- Run integration tests:
- In stack-orchestrator repo:
- Create config file:
```bash
cd helper-scripts
./create-config.sh
```
A `config.sh` will be created in the root directory.
- Update/Edit the config file `config.sh`:
```bash
#!/bin/bash
# Path to ipld-eth-server repo.
vulcanize_ipld_eth_db=~/ipld-eth-db/
# Path to go-ethereum repo.
vulcanize_go_ethereum=~/go-ethereum
# Path to contract folder.
vulcanize_test_contract=~/ipld-eth-db-validator/test/contract
genesis_file_path='start-up-files/go-ethereum/genesis.json'
db_write=true
```
- Run stack-orchestrator:
```bash
# In stack-orchestrator root directory.
cd helper-scripts
./wrapper.sh \
-e docker \
-d ../docker/local/docker-compose-db-sharding.yml \
-d ../docker/local/docker-compose-go-ethereum.yml \
-d ../docker/local/docker-compose-contract.yml \
-v remove \
-p ../config.sh
```
- Run tests:
```bash
# In ipld-eth-db-validator root directory.
./scripts/run_integration_test.sh
```

View File

@ -1,19 +0,0 @@
# Runs the test contract deployment server
services:
contract-deployer:
restart: on-failure
image: cerc/ipld-eth-db-validator/test-contract-deployer:local
build: ./contract
networks:
- test_default
environment:
ETH_ADDR: "http://fixturenet-eth-geth-1:8545"
ETH_CHAIN_ID: $ETH_CHAIN_ID
DEPLOYER_PRIVATE_KEY: $DEPLOYER_PRIVATE_KEY
ports:
- 127.0.0.1:3000:3000
networks:
test_default:
external: true

View File

@ -1,5 +1,7 @@
# Downgrade from 18.16, see https://github.com/NomicFoundation/hardhat/issues/3877
FROM node:20-slim
FROM node:14
ARG ETH_ADDR
ENV ETH_ADDR $ETH_ADDR
WORKDIR /usr/src/app
COPY package*.json ./
@ -9,4 +11,4 @@ RUN npm run compile && ls -lah
EXPOSE 3000
ENTRYPOINT ["node", "src/index.js"]
ENTRYPOINT ["npm", "start"]

View File

@ -1,16 +1,10 @@
pragma solidity ^0.8.25;
pragma solidity ^0.8.0;
import "@openzeppelin/contracts/token/ERC20/ERC20.sol";
contract GLDToken is ERC20 {
constructor() ERC20("Gold", "GLD") {
_mint(msg.sender, 1000000000000000000000);
}
function destroy() public {
(bool ok, ) = payable(msg.sender).call{value: address(this).balance}("");
require(ok, "ETH transfer failed");
_burn(msg.sender, balanceOf(msg.sender));
selfdestruct(payable(msg.sender));
}
}

View File

@ -2,28 +2,28 @@
pragma solidity ^0.8.0;
contract Test {
event logPut (address, uint256);
address payable owner;
mapping(address => uint256) public data;
modifier onlyOwner {
require(msg.sender == owner, "Only owner can call this function.");
require(
msg.sender == owner,
"Only owner can call this function."
);
_;
}
uint256[100] data;
constructor() {
owner = payable(msg.sender);
data = [1];
}
function Put(uint256 value) public {
emit logPut(msg.sender, value);
data[msg.sender] = value;
function Put(uint256 addr, uint256 value) public {
data[addr] = value;
}
function close() public onlyOwner {
(bool ok, ) = owner.call{value: address(this).balance}("");
require(ok, "ETH transfer failed");
selfdestruct(owner);
}
}

View File

@ -16,31 +16,17 @@ task("accounts", "Prints the list of accounts", async () => {
/**
* @type import('hardhat/config').HardhatUserConfig
*/
const localNetwork = {
url: process.env.ETH_ADDR || "http://127.0.0.1:8545",
chainId: Number(process.env.ETH_CHAIN_ID) || 99,
};
if (process.env.DEPLOYER_PRIVATE_KEY) {
localNetwork["accounts"] = [process.env.DEPLOYER_PRIVATE_KEY];
}
module.exports = {
solidity: {
version: "0.8.25",
settings: {
outputSelection: {
'*': {
'*': [
'abi', 'storageLayout',
]
}
}
}
},
solidity: "0.8.0",
networks: {
local: localNetwork
local: {
url: 'http://127.0.0.1:8545',
chainId: 99
},
defaultNetwork: "local"
docker: {
url: process.env.ETH_ADDR,
chainId: 99
}
}
};

File diff suppressed because it is too large Load Diff

View File

@ -4,21 +4,22 @@
"main": "index.js",
"scripts": {
"compile": "npx hardhat compile",
"start": "node src/index.js"
"start": "HARDHAT_NETWORK=docker node src/index.js",
"start:local": "ETH_ADDR=http://127.0.0.1:8545 npm run start",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"description": "Solidity contract deployment server for integration testing",
"description": "",
"dependencies": {
"@openzeppelin/contracts": "^5.0.2",
"fastify": "^4.26.2",
"hardhat": "^2.22.3",
"solidity-create2-deployer": "^0.4.0"
"@openzeppelin/contracts": "^4.0.0",
"fastify": "^3.14.2",
"hardhat": "^2.2.0"
},
"devDependencies": {
"@nomiclabs/hardhat-ethers": "^2.2.3",
"@nomiclabs/hardhat-waffle": "^2.0.4",
"@nomiclabs/hardhat-ethers": "^2.0.2",
"@nomiclabs/hardhat-waffle": "^2.0.1",
"chai": "^4.3.4",
"ethereum-waffle": "^3.3.0",
"ethers": "^5.1.0"

View File

@ -0,0 +1,18 @@
const hre = require("hardhat");
async function main() {
// await hre.run('compile');
// We get the contract to deploy
const GLDToken = await hre.ethers.getContractFactory("GLDToken");
const token = await GLDToken.deploy();
await token.deployed();
console.log("GLDToken deployed to:", token.address, token.deployTransaction.hash);
}
// We recommend this pattern to be able to use async/await everywhere
// and properly handle errors.
main()
.then(() => process.exit(0))
.catch(error => {
console.error(error);
process.exit(1);
});

View File

@ -0,0 +1,36 @@
// We require the Hardhat Runtime Environment explicitly here. This is optional
// but useful for running the script in a standalone fashion through `node <script>`.
//
// When running the script with `hardhat run <script>` you'll find the Hardhat
// Runtime Environment's members available in the global scope.
const hre = require("hardhat");
async function main() {
// Hardhat always runs the compile task when running scripts with its command
// line interface.
//
// If this script is run directly using `node` you may want to call compile
// manually to make sure everything is compiled
// await hre.run('compile');
// We get the contract to deploy
const Greeter = await hre.ethers.getContractFactory("Greeter");
const greeter = await Greeter.deploy("Hello, Hardhat!");
await greeter.deployed();
console.log("Greeter deployed to:", greeter.address, "; tx hash: ", greeter.deployTransaction.hash);
const result = await greeter.setGreeting("Hello 123!");
console.log("Greeter updated", "; tx hash: ", result.hash, "; block hash: ", result.blockHash);
}
// We recommend this pattern to be able to use async/await everywhere
// and properly handle errors.
main()
.then(() => process.exit(0))
.catch(error => {
console.error(error);
process.exit(1);
});

View File

@ -1,6 +1,7 @@
const fastify = require('fastify')({ logger: true });
const hre = require("hardhat");
// readiness check
fastify.get('/v1/healthz', async (req, reply) => {
reply
@ -9,79 +10,109 @@ fastify.get('/v1/healthz', async (req, reply) => {
.send({ success: true })
});
fastify.get('/v1/deployContract', async (req, reply) => {
const GLDToken = await hre.ethers.getContractFactory("GLDToken");
const token = await GLDToken.deploy();
await token.deployed();
return {
address: token.address,
txHash: token.deployTransaction.hash,
blockNumber: token.deployTransaction.blockNumber,
blockHash: token.deployTransaction.blockHash,
}
});
fastify.get('/v1/destroyContract', async (req, reply) => {
const addr = req.query.addr;
const Token = await hre.ethers.getContractFactory("GLDToken");
const token = await Token.attach(addr);
await token.destroy();
const blockNum = await hre.ethers.provider.getBlockNumber()
return {
blockNumber: blockNum,
}
})
fastify.get('/v1/sendEth', async (req, reply) => {
const to = req.query.to;
const value = hre.ethers.utils.parseEther(req.query.value);
const value = req.query.value;
const owner = await hre.ethers.getSigner();
const tx = await owner.sendTransaction({to, value}).then(tx => tx.wait());
const [owner] = await hre.ethers.getSigners();
const tx = await owner.sendTransaction({
to,
value: hre.ethers.utils.parseEther(value)
});
await tx.wait(1)
// console.log(tx);
// const coinbaseBalance = await hre.ethers.provider.getBalance(owner.address);
// const receiverBalance = await hre.ethers.provider.getBalance(to);
// console.log(coinbaseBalance.toString(), receiverBalance.toString());
return {
from: tx.from,
to: tx.to,
//value: tx.value.toString(),
txHash: tx.hash,
blockNumber: tx.blockNumber,
blockHash: tx.blockHash,
}
});
function contractCreator(name) {
return async (req, reply) => {
const contract = await hre.ethers.getContractFactory(name);
const instance = await contract.deploy();
const rct = await instance.deployTransaction.wait();
fastify.get('/v1/deployTestContract', async (req, reply) => {
const testContract = await hre.ethers.getContractFactory("Test");
const test = await testContract.deploy();
await test.deployed();
return {
address: instance.address,
txHash: rct.transactionHash,
blockNumber: rct.blockNumber,
blockHash: rct.blockHash,
address: test.address,
txHash: test.deployTransaction.hash,
blockNumber: test.deployTransaction.blockNumber,
blockHash: test.deployTransaction.blockHash,
}
}
}
function contractDestroyer(name) {
return async (req, reply) => {
const addr = req.query.addr;
const contract = await hre.ethers.getContractFactory(name);
const instance = contract.attach(addr);
const rct = await instance.destroy().then(tx => tx.wait());
return {
blockNumber: rct.blockNumber,
}
}
}
fastify.get('/v1/deployContract', contractCreator("GLDToken"));
fastify.get('/v1/destroyContract', contractDestroyer("GLDToken"));
fastify.get('/v1/deployTestContract', contractCreator("Test"));
fastify.get('/v1/destroyTestContract', contractDestroyer("Test"));
});
fastify.get('/v1/putTestValue', async (req, reply) => {
const addr = req.query.addr;
const index = req.query.index;
const value = req.query.value;
const testContract = await hre.ethers.getContractFactory("Test");
const test = await testContract.attach(addr);
const rct = await test.Put(value).then(tx => tx.wait());
const tx = await test.Put(index, value);
const receipt = await tx.wait();
return {
blockNumber: rct.blockNumber,
blockNumber: receipt.blockNumber,
}
});
fastify.get('/v1/destroyTestContract', async (req, reply) => {
const addr = req.query.addr;
const testContract = await hre.ethers.getContractFactory("Test");
const test = await testContract.attach(addr);
await test.destroy();
const blockNum = await hre.ethers.provider.getBlockNumber()
return {
blockNumber: blockNum,
}
})
async function main() {
try {
await fastify.listen({ port: 3000, host: '0.0.0.0' });
await fastify.listen(3000, '0.0.0.0');
} catch (err) {
fastify.log.error(err);
process.exit(1);
}
}
process.on('SIGINT', () => fastify.close().then(() => process.exit(1)));
main();

View File

@ -1,13 +0,0 @@
import { expect } from 'chai';
describe("GLDToken", function() {
it("Should return the owner's balance", async function() {
const Token = await ethers.getContractFactory("GLDToken");
const token = await Token.deploy();
await token.deployed();
const [owner] = await ethers.getSigners();
const balance = await token.balanceOf(owner.address);
expect(balance).to.equal('1000000000000000000000');
expect(await token.totalSupply()).to.equal(balance); });
});

View File

@ -0,0 +1,14 @@
const { expect } = require("chai");
describe("Greeter", function() {
it("Should return the new greeting once it's changed", async function() {
const Greeter = await ethers.getContractFactory("Greeter");
const greeter = await Greeter.deploy("Hello, world!");
await greeter.deployed();
expect(await greeter.greet()).to.equal("Hello, world!");
await greeter.setGreeting("Hola, mundo!");
expect(await greeter.greet()).to.equal("Hola, mundo!");
});
});

54
test/helper.go Normal file
View File

@ -0,0 +1,54 @@
package integration
import (
"encoding/json"
"fmt"
"net/http"
ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
)
type PutResult struct {
BlockNumber int64 `json:"blockNumber"`
}
const srvUrl = "http://localhost:3000"
func DeployTestContract() (*ethServerIntegration.ContractDeployed, error) {
ethServerIntegration.DeployContract()
res, err := http.Get(fmt.Sprintf("%s/v1/deployTestContract", srvUrl))
if err != nil {
return nil, err
}
defer res.Body.Close()
var contract ethServerIntegration.ContractDeployed
decoder := json.NewDecoder(res.Body)
return &contract, decoder.Decode(&contract)
}
func PutTestValue(addr string, index, value int) (*PutResult, error) {
res, err := http.Get(fmt.Sprintf("%s/v1/putTestValue?addr=%s&index=%d&value=%d", srvUrl, addr, index, value))
if err != nil {
return nil, err
}
var blockNumber PutResult
decoder := json.NewDecoder(res.Body)
return &blockNumber, decoder.Decode(&blockNumber)
}
func DestroyTestContract(addr string) (*ethServerIntegration.ContractDestroyed, error) {
res, err := http.Get(fmt.Sprintf("%s/v1/destroyTestContract?addr=%s", srvUrl, addr))
if err != nil {
return nil, err
}
defer res.Body.Close()
var data ethServerIntegration.ContractDestroyed
decoder := json.NewDecoder(res.Body)
return &data, decoder.Decode(&data)
}

View File

@ -0,0 +1,19 @@
package integration_test
import (
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestIntegration(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "integration test suite")
}
var _ = BeforeSuite(func() {
logrus.SetOutput(ioutil.Discard)
})

108
test/integration_test.go Normal file
View File

@ -0,0 +1,108 @@
package integration_test
import (
"context"
"sync"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
integration "github.com/vulcanize/ipld-eth-db-validator/test"
"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
ethServerIntegration "github.com/vulcanize/ipld-eth-server/v4/test"
)
const (
blockNum = 1
trail = 0
validatorSleepInterval = uint(5)
)
var (
testAddresses = []string{
"0x1111111111111111111111111111111111111112",
"0x1ca7c995f8eF0A2989BbcE08D5B7Efe50A584aa1",
"0x9a4b666af23a2cdb4e5538e1d222a445aeb82134",
"0xF7C7AEaECD2349b129d5d15790241c32eeE4607B",
"0x992b6E9BFCA1F7b0797Cee10b0170E536EAd3532",
"0x29ed93a7454Bc17a8D4A24D0627009eE0849B990",
"0x66E3dCA826b04B5d4988F7a37c91c9b1041e579D",
"0x96288939Ac7048c27E0E087b02bDaad3cd61b37b",
"0xD354280BCd771541c935b15bc04342c26086FE9B",
"0x7f887e25688c274E77b8DeB3286A55129B55AF14",
}
)
var _ = Describe("Integration test", func() {
ctx := context.Background()
var contract *ethServerIntegration.ContractDeployed
var err error
sleepInterval := 2 * time.Second
timeout := 4 * time.Second
db := shared.SetupDB()
cfg := validator.Config{
DB: db,
BlockNum: blockNum,
Trail: trail,
SleepInterval: validatorSleepInterval,
ChainCfg: validator.IntegrationTestChainConfig,
}
validationProgressChan := make(chan uint64)
service := validator.NewService(&cfg, validationProgressChan)
wg := new(sync.WaitGroup)
It("test init", func() {
wg.Add(1)
go service.Start(ctx, wg)
// Deploy a dummy contract as the first contract might get deployed at block number 0
_, _ = ethServerIntegration.DeployContract()
time.Sleep(sleepInterval)
})
defer It("test teardown", func() {
service.Stop()
wg.Wait()
Expect(validationProgressChan).To(BeClosed())
})
Describe("Validate state", func() {
It("performs validation on contract deployment", func() {
contract, err = integration.DeployTestContract()
Expect(err).ToNot(HaveOccurred())
time.Sleep(sleepInterval)
Expect(validationProgressChan).ToNot(BeClosed())
Eventually(validationProgressChan, timeout).Should(Receive(Equal(uint64(contract.BlockNumber))))
})
It("performs validation on contract transactions", func() {
for i := 0; i < 10; i++ {
res, txErr := integration.PutTestValue(contract.Address, i, i)
Expect(txErr).ToNot(HaveOccurred())
time.Sleep(sleepInterval)
Expect(validationProgressChan).ToNot(BeClosed())
Eventually(validationProgressChan, timeout).Should(Receive(Equal(uint64(res.BlockNumber))))
}
})
It("performs validation on eth transfer transactions", func() {
for _, address := range testAddresses {
tx, txErr := ethServerIntegration.SendEth(address, "0.01")
Expect(txErr).ToNot(HaveOccurred())
time.Sleep(sleepInterval)
Expect(validationProgressChan).ToNot(BeClosed())
Eventually(validationProgressChan, timeout).Should(Receive(Equal(uint64(tx.BlockNumber))))
}
})
})
})

View File

@ -0,0 +1,117 @@
package validator_test
import (
"crypto/ecdsa"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
)
// Test variables
var (
Testdb = rawdb.NewMemoryDatabase()
TestBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
TestBankFunds = big.NewInt(100000000)
Genesis = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
DeploymentTxData = common.Hex2Bytes("0x608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
ContractCode = common.Hex2Bytes("0x608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
CodeHash = crypto.Keccak256Hash(ContractCode)
ContractAddr common.Address
IndexZero = "0000000000000000000000000000000000000000000000000000000000000000"
IndexOne = "0000000000000000000000000000000000000000000000000000000000000001"
ContractSlotPosition = common.FromHex(IndexOne)
ContractSlotKeyHash = crypto.Keccak256Hash(ContractSlotPosition)
MiningReward = big.NewInt(2000000000000000000)
)
/* test function signatures
put function sig: 65f3c31a
close function sig: 43d726d6
data function sig: 73d4a13a
*/
func createLondonTransaction(block *core.BlockGen, addr *common.Address, key *ecdsa.PrivateKey) *types.Transaction {
londonTrx := types.NewTx(&types.DynamicFeeTx{
ChainID: validator.TestChainConfig.ChainID,
Nonce: block.TxNonce(*addr),
GasTipCap: big.NewInt(50),
GasFeeCap: big.NewInt(1000000000),
Gas: 21000,
To: addr,
Value: big.NewInt(1000),
Data: []byte{},
})
transactionSigner := types.MakeSigner(validator.TestChainConfig, block.Number())
signedTrx1, err := types.SignTx(londonTrx, transactionSigner, key)
if err != nil {
log.Fatal(err.Error())
}
return signedTrx1
}
// MakeChain creates a chain of n blocks starting at and including parent.
// the returned hash chain is ordered head->parent.
func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, []types.Receipts, *core.BlockChain) {
config := validator.TestChainConfig
blocks, receipts := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
chain, _ := core.NewBlockChain(Testdb, nil, validator.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
return append([]*types.Block{parent}, blocks...), receipts, chain
}
func TestChainGen(i int, block *core.BlockGen) {
signer := types.HomesteadSigner{}
switch i {
case 0:
// In block 1, the test bank sends account #1 some ether.
tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey)
block.AddTx(tx)
case 1:
// In block 2, the test bank sends some more ether to account #1.
// Account1Addr passes it on to account #2.
// Account1Addr creates a test contract.
tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey)
nonce := block.TxNonce(Account1Addr)
tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
nonce++
tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), DeploymentTxData), signer, Account1Key)
ContractAddr = crypto.CreateAddress(Account1Addr, nonce)
block.AddTx(tx1)
block.AddTx(tx2)
block.AddTx(tx3)
case 2:
block.SetCoinbase(Account2Addr)
data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000003")
tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
block.AddTx(tx)
case 3:
block.SetCoinbase(Account2Addr)
data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000009")
tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
block.AddTx(tx)
case 4:
block.SetCoinbase(Account1Addr)
data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000000")
tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
block.AddTx(tx)
case 5:
block.AddTx(createLondonTransaction(block, &Account1Addr, Account1Key))
case 6:
block.AddTx(createLondonTransaction(block, &Account2Addr, Account2Key))
}
}

View File

@ -0,0 +1,19 @@
package validator_test_test
import (
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
func TestETHSuite(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "eth ipld validator eth suite test")
}
var _ = BeforeSuite(func() {
logrus.SetOutput(ioutil.Discard)
})

View File

@ -0,0 +1,129 @@
package validator_test_test
import (
"context"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/statediff"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/jmoiron/sqlx"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipld-eth-server/v4/pkg/eth/test_helpers"
"github.com/vulcanize/ipld-eth-server/v4/pkg/shared"
"github.com/vulcanize/ipld-eth-db-validator/pkg/validator"
"github.com/vulcanize/ipld-eth-db-validator/validator_test"
)
const (
chainLength = 20
blockHeight = 1
trail = 2
)
var _ = Describe("eth state reading tests", func() {
var (
blocks []*types.Block
receipts []types.Receipts
chain *core.BlockChain
db *sqlx.DB
chainConfig = validator.TestChainConfig
mockTD = big.NewInt(1337)
)
It("test init", func() {
db = shared.SetupDB()
transformer := shared.SetupTestStateDiffIndexer(context.Background(), chainConfig, validator_test.Genesis.Hash())
// make the test blockchain (and state)
blocks, receipts, chain = validator_test.MakeChain(chainLength, validator_test.Genesis, validator_test.TestChainGen)
params := statediff.Params{
IntermediateStateNodes: true,
IntermediateStorageNodes: true,
}
// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
builder := statediff.NewBuilder(chain.StateCache())
for i, block := range blocks {
var args statediff.Args
var rcts types.Receipts
if i == 0 {
args = statediff.Args{
OldStateRoot: common.Hash{},
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
} else {
args = statediff.Args{
OldStateRoot: blocks[i-1].Root(),
NewStateRoot: block.Root(),
BlockNumber: block.Number(),
BlockHash: block.Hash(),
}
rcts = receipts[i-1]
}
diff, err := builder.BuildStateDiffObject(args, params)
Expect(err).ToNot(HaveOccurred())
tx, err := transformer.PushBlock(block, rcts, mockTD)
Expect(err).ToNot(HaveOccurred())
for _, node := range diff.Nodes {
err := transformer.PushStateNode(tx, node, block.Hash().String())
Expect(err).ToNot(HaveOccurred())
}
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
}
// Insert some non-canonical data into the database so that we test our ability to discern canonicity
indexAndPublisher := shared.SetupTestStateDiffIndexer(context.Background(), chainConfig, validator_test.Genesis.Hash())
tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
// The non-canonical header has a child
tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
Expect(err).ToNot(HaveOccurred())
hash := sdtypes.CodeAndCodeHash{
Hash: test_helpers.CodeHash,
Code: test_helpers.ContractCode,
}
err = indexAndPublisher.PushCodeAndCodeHash(tx, hash)
Expect(err).ToNot(HaveOccurred())
err = tx.Submit(err)
Expect(err).ToNot(HaveOccurred())
})
defer It("test teardown", func() {
shared.TearDownDB(db)
chain.Stop()
})
Describe("state_validation", func() {
It("Validator", func() {
api, err := validator.EthAPI(context.Background(), db, validator.TestChainConfig)
Expect(err).ToNot(HaveOccurred())
for i := uint64(blockHeight); i <= chainLength-trail; i++ {
err = validator.ValidateBlock(context.Background(), api, i)
Expect(err).ToNot(HaveOccurred())
err = validator.ValidateReferentialIntegrity(db, i)
Expect(err).ToNot(HaveOccurred())
}
})
})
})