Compare commits

..

1 Commits
v5 ... master

Author SHA1 Message Date
erikdies
94640a8c78
Create issues-notion-sync.yml 2022-06-29 08:46:38 -04:00
18 changed files with 1797 additions and 1388 deletions

View File

@ -0,0 +1,29 @@
name: Notion Sync
on:
workflow_dispatch:
issues:
types:
[
opened,
edited,
labeled,
unlabeled,
assigned,
unassigned,
milestoned,
demilestoned,
reopened,
closed,
]
jobs:
notion_job:
runs-on: ubuntu-latest
name: Add GitHub Issues to Notion
steps:
- name: Add GitHub Issues to Notion
uses: vulcanize/notion-github-action@v1.2.4-issueid
with:
notion-token: ${{ secrets.NOTION_TOKEN }}
notion-db: ${{ secrets.NOTION_DATABASE }}

View File

@ -1,38 +0,0 @@
name: Run tests
on:
- pull_request
env:
# Needed until we can incorporate docker startup into the executor container
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
unit-tests:
name: Run unit tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version-file: 'go.mod'
check-latest: true
- name: Run dockerd
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: Run DB container
working-directory: ./test
run: docker compose up --wait --quiet-pull
- name: Set Gitea access token
env:
TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
run: |
git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf https://git.vdb.to/
- name: Build and run tests
run: |
until [[ "$(docker inspect -f '{{.State.Status}}' test-ipld-eth-db)" = 'running' ]]
do sleep 1; done &
go build ./...
wait $!
go test -p 1 -v ./...

View File

@ -1,8 +1,8 @@
# eth-ipfs-state-validator
[![Go Report Card](https://goreportcard.com/badge/github.com/cerc-io/eth-ipfs-state-validator)](https://goreportcard.com/report/github.com/cerc-io/eth-ipfs-state-validator)
[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-ipfs-state-validator)](https://goreportcard.com/report/github.com/vulcanize/eth-ipfs-state-validator)
> Uses [ipfs-ethdb](https://github.com/cerc-io/ipfs-ethdb/tree/master/postgres) to validate completeness of IPFS Ethereum state data
> Uses [ipfs-ethdb](https://github.com/vulcanize/ipfs-ethdb/tree/master/postgres) to validate completeness of IPFS Ethereum state data
## Background
@ -42,7 +42,7 @@ Postgres DB config:
```toml
[database]
name = "cerc_public"
name = "vulcanize_public"
hostname = "localhost"
user = "postgres"
password = ""
@ -50,7 +50,7 @@ Postgres DB config:
```
## Maintainers
@cerc-io
@vulcanize
@AFDudley
@i-norden
@ -60,4 +60,4 @@ Contributions are welcome!
VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
## License
[AGPL-3.0](LICENSE) © Vulcanize Inc
[AGPL-3.0](LICENSE) © Vulcanize Inc

View File

@ -85,7 +85,7 @@ func init() {
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
rootCmd.PersistentFlags().String("database-name", "cerc_public", "database name")
rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
rootCmd.PersistentFlags().String("database-user", "", "database user")

View File

@ -24,7 +24,7 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
validator "github.com/cerc-io/eth-ipfs-state-validator/v5/pkg"
validator "github.com/vulcanize/eth-ipfs-state-validator/v3/pkg"
)
// validateTrieCmd represents the validateTrie command
@ -59,41 +59,32 @@ It can operate at three levels:
}
func validateTrie() {
params := validator.Params{
Workers: viper.GetUint("validator.workers"),
RecoveryFormat: viper.GetString("validator.recoveryFormat"),
}
v, err := newValidator(params)
v, err := newValidator()
if err != nil {
logWithCommand.Fatal(err)
}
stateRootStr := viper.GetString("validator.stateRoot")
storageRootStr := viper.GetString("validator.storageRoot")
contractAddrStr := viper.GetString("validator.address")
if stateRootStr == "" {
logWithCommand.Fatal("must provide a state root for state trie validation")
}
stateRoot := common.HexToHash(stateRootStr)
traversal := strings.ToLower(viper.GetString("validator.type"))
switch traversal {
switch strings.ToLower(viper.GetString("validator.type")) {
case "f", "full":
logWithCommand.
WithField("root", stateRoot).
Debug("Validating full state")
if stateRootStr == "" {
logWithCommand.Fatal("must provide a state root for full state validation")
}
stateRoot := common.HexToHash(stateRootStr)
if err = v.ValidateTrie(stateRoot); err != nil {
logWithCommand.Fatalf("Validation failed: %v", err)
logWithCommand.Fatalf("State for root %s is not complete\r\nerr: %v", stateRoot.String(), err)
}
logWithCommand.Infof("State for root %s is complete", stateRoot)
logWithCommand.Infof("State for root %s is complete", stateRoot.String())
case "state":
logWithCommand.
WithField("root", stateRoot).
Debug("Validating state trie")
if err = v.ValidateStateTrie(stateRoot); err != nil {
logWithCommand.Fatalf("Validation failed: %s", err)
if stateRootStr == "" {
logWithCommand.Fatal("must provide a state root for state trie validation")
}
logWithCommand.Infof("State trie for root %s is complete", stateRoot)
stateRoot := common.HexToHash(stateRootStr)
if err = v.ValidateStateTrie(stateRoot); err != nil {
logWithCommand.Fatalf("State trie for root %s is not complete\r\nerr: %v", stateRoot.String(), err)
}
logWithCommand.Infof("State trie for root %s is complete", stateRoot.String())
case "storage":
if storageRootStr == "" {
logWithCommand.Fatal("must provide a storage root for storage trie validation")
@ -103,36 +94,30 @@ func validateTrie() {
}
storageRoot := common.HexToHash(storageRootStr)
addr := common.HexToAddress(contractAddrStr)
logWithCommand.
WithField("contract", addr).
WithField("storage root", storageRoot).
Debug("Validating storage trie")
if err = v.ValidateStorageTrie(stateRoot, addr, storageRoot); err != nil {
logWithCommand.Fatal("Validation failed", err)
if err = v.ValidateStorageTrie(addr, storageRoot); err != nil {
logWithCommand.Fatalf("Storage trie for contract %s and root %s not complete\r\nerr: %v", addr.String(), storageRoot.String(), err)
}
logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr, storageRoot)
default:
logWithCommand.Fatal("Invalid traversal level:", traversal)
logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String())
}
stats := v.GetCacheStats()
logWithCommand.Debugf("groupcache stats %+v", stats)
}
func newValidator(params validator.Params) (*validator.Validator, error) {
func newValidator() (*validator.Validator, error) {
ipfsPath := viper.GetString("ipfs.path")
if ipfsPath == "" {
db, err := validator.NewDB()
if err != nil {
logWithCommand.Fatal(err)
}
return validator.NewPGIPFSValidator(db, params), nil
return validator.NewPGIPFSValidator(db), nil
}
bs, err := validator.InitIPFSBlockService(ipfsPath)
if err != nil {
return nil, err
}
return validator.NewIPFSValidator(bs, params), nil
return validator.NewIPFSValidator(bs), nil
}
func init() {
@ -143,14 +128,10 @@ func init() {
validateTrieCmd.PersistentFlags().String("storage-root", "", "Root of the storage trie we wish to validate; for storage validation")
validateTrieCmd.PersistentFlags().String("address", "", "Contract address for the storage trie we wish to validate; for storage validation")
validateTrieCmd.PersistentFlags().String("ipfs-path", "", "Path to IPFS repository; if provided operations move through the IPFS repo otherwise Postgres connection params are expected in the provided config")
validateTrieCmd.PersistentFlags().Int("workers", 4, "number of concurrent workers to use")
validateTrieCmd.PersistentFlags().String("recovery-format", validator.DefaultRecoveryFormat, "format pattern for recovery files")
viper.BindPFlag("validator.stateRoot", validateTrieCmd.PersistentFlags().Lookup("state-root"))
viper.BindPFlag("validator.type", validateTrieCmd.PersistentFlags().Lookup("type"))
viper.BindPFlag("validator.storageRoot", validateTrieCmd.PersistentFlags().Lookup("storage-root"))
viper.BindPFlag("validator.address", validateTrieCmd.PersistentFlags().Lookup("address"))
viper.BindPFlag("validator.workers", validateTrieCmd.PersistentFlags().Lookup("workers"))
viper.BindPFlag("validator.recoveryFormat", validateTrieCmd.PersistentFlags().Lookup("recovery-format"))
viper.BindPFlag("ipfs.path", validateTrieCmd.PersistentFlags().Lookup("ipfs-path"))
}

View File

@ -1,5 +1,5 @@
[database]
name = "cerc_public"
name = "vulcanize_public"
hostname = "localhost"
port = 5432
user = "postgres"

View File

@ -1,4 +1,4 @@
[database]
name = "cerc_testing"
name = "vulcanize_testing"
hostname = "localhost"
port = 5432

407
go.mod
View File

@ -1,265 +1,238 @@
module github.com/cerc-io/eth-ipfs-state-validator/v5
module github.com/vulcanize/eth-ipfs-state-validator/v3
go 1.21
go 1.18
require (
github.com/cerc-io/eth-iterator-utils v0.2.0
github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha
github.com/cerc-io/ipld-eth-statedb v0.1.0
github.com/ethereum/go-ethereum v1.13.14
github.com/holiman/uint256 v1.2.4
github.com/ipfs/boxo v0.19.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/kubo v0.27.0
github.com/ethereum/go-ethereum v1.10.18
github.com/ipfs/go-blockservice v0.1.7
github.com/ipfs/go-cid v0.0.7
github.com/ipfs/go-ipfs v0.10.0
github.com/ipfs/go-ipfs-blockstore v1.0.1
github.com/ipfs/go-ipfs-ds-help v1.0.0
github.com/jmoiron/sqlx v1.3.5
github.com/lib/pq v1.10.9
github.com/lib/pq v1.10.5
github.com/mailgun/groupcache/v2 v2.3.0
github.com/multiformats/go-multihash v0.2.3
github.com/onsi/ginkgo/v2 v2.15.0
github.com/onsi/gomega v1.30.0
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.18.2
golang.org/x/sync v0.7.0
github.com/multiformats/go-multihash v0.1.0
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.19.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.4.0
github.com/spf13/viper v1.11.0
github.com/vulcanize/ipfs-ethdb/v3 v3.0.3
)
require (
bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect
github.com/DataDog/zstd v1.5.5 // indirect
github.com/Jorropo/jsync v1.0.1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
github.com/Stebalien/go-bitfield v0.0.1 // indirect
github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.10.0 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cerc-io/plugeth-statediff v0.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cockroachdb/errors v1.10.0 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/consensys/bavard v0.1.13 // indirect
github.com/consensys/gnark-crypto v0.12.1 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
github.com/btcsuite/btcd v0.22.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/cskr/pubsub v1.0.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/deckarep/golang-set/v2 v2.3.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
github.com/flynn/noise v1.1.0 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect
github.com/georgysavva/scany v0.2.9 // indirect
github.com/getsentry/sentry-go v0.22.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-stack/stack v1.8.1 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
github.com/huin/goupnp v1.3.0 // indirect
github.com/inconshreveable/log15 v2.16.0+incompatible // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/huin/goupnp v1.0.3 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitfield v1.1.0 // indirect
github.com/ipfs/go-block-format v0.2.0 // indirect
github.com/ipfs/go-cidutil v0.1.0 // indirect
github.com/ipfs/go-datastore v0.6.0 // indirect
github.com/ipfs/go-ds-measure v0.2.0 // indirect
github.com/ipfs/go-bitswap v0.4.0 // indirect
github.com/ipfs/go-block-format v0.0.3 // indirect
github.com/ipfs/go-cidutil v0.0.2 // indirect
github.com/ipfs/go-datastore v0.4.6 // indirect
github.com/ipfs/go-ds-measure v0.1.0 // indirect
github.com/ipfs/go-fetcher v1.5.0 // indirect
github.com/ipfs/go-filestore v1.0.0 //indirect
github.com/ipfs/go-fs-lock v0.0.7 // indirect
github.com/ipfs/go-graphsync v0.8.0 // indirect
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
github.com/ipfs/go-ipfs-config v0.16.0 // indirect
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
github.com/ipfs/go-ipfs-util v0.0.3 // indirect
github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
github.com/ipfs/go-ipld-format v0.6.0 // indirect
github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.0.1 // indirect
github.com/ipfs/go-ipfs-exchange-offline v0.0.1 // indirect
github.com/ipfs/go-ipfs-files v0.0.8 // indirect
github.com/ipfs/go-ipfs-keystore v0.0.2 // indirect
github.com/ipfs/go-ipfs-pinner v0.1.2 // indirect
github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
github.com/ipfs/go-ipfs-provider v0.6.1 // indirect
github.com/ipfs/go-ipfs-routing v0.1.0 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
github.com/ipfs/go-ipld-format v0.2.0 // indirect
github.com/ipfs/go-ipld-legacy v0.1.0 // indirect
github.com/ipfs/go-ipns v0.1.2 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-log/v2 v2.3.0 // indirect
github.com/ipfs/go-merkledag v0.4.0 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
github.com/ipfs/go-unixfsnode v1.9.0 // indirect
github.com/ipld/go-car/v2 v2.13.1 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/ipld/go-ipld-prime v0.21.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.14.3 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.3 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgtype v1.14.0 // indirect
github.com/jackc/pgx/v4 v4.18.3 // indirect
github.com/jackc/puddle v1.3.0 // indirect
github.com/ipfs/go-mfs v0.1.2 // indirect
github.com/ipfs/go-namesys v0.3.1 // indirect
github.com/ipfs/go-path v0.1.2 // indirect
github.com/ipfs/go-peertaskqueue v0.4.0 // indirect
github.com/ipfs/go-unixfs v0.2.5 // indirect
github.com/ipfs/go-unixfsnode v1.1.3 // indirect
github.com/ipfs/go-verifcid v0.0.1 // indirect
github.com/ipfs/interface-go-ipfs-core v0.5.1 // indirect
github.com/ipld/go-codec-dagpb v1.3.0 // indirect
github.com/ipld/go-ipld-prime v0.12.2 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/klauspost/compress v1.17.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/klauspost/compress v1.11.7 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/koron/go-ssdp v0.0.2 // indirect
github.com/libp2p/go-addr-util v0.1.0 // indirect
github.com/libp2p/go-buffer-pool v0.0.2 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-doh-resolver v0.4.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
github.com/libp2p/go-libp2p v0.33.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.24.4 // indirect
github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
github.com/libp2p/go-libp2p-pubsub v0.10.0 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.3 // indirect
github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
github.com/libp2p/go-msgio v0.3.0 // indirect
github.com/libp2p/go-nat v0.2.0 // indirect
github.com/libp2p/go-netroute v0.2.1 // indirect
github.com/libp2p/go-reuseport v0.4.0 // indirect
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/libp2p/go-conn-security-multistream v0.2.1 // indirect
github.com/libp2p/go-doh-resolver v0.3.1 // indirect
github.com/libp2p/go-eventbus v0.2.1 // indirect
github.com/libp2p/go-flow-metrics v0.0.3 // indirect
github.com/libp2p/go-libp2p v0.15.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 // indirect
github.com/libp2p/go-libp2p-autonat v0.4.2 // indirect
github.com/libp2p/go-libp2p-blankhost v0.2.0 // indirect
github.com/libp2p/go-libp2p-circuit v0.4.0 // indirect
github.com/libp2p/go-libp2p-connmgr v0.2.4 // indirect
github.com/libp2p/go-libp2p-core v0.9.0 // indirect
github.com/libp2p/go-libp2p-discovery v0.5.1 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.13.1 // indirect
github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect
github.com/libp2p/go-libp2p-loggables v0.1.0 // indirect
github.com/libp2p/go-libp2p-mplex v0.4.1 // indirect
github.com/libp2p/go-libp2p-nat v0.0.6 // indirect
github.com/libp2p/go-libp2p-noise v0.2.2 // indirect
github.com/libp2p/go-libp2p-peerstore v0.2.8 // indirect
github.com/libp2p/go-libp2p-pnet v0.2.0 // indirect
github.com/libp2p/go-libp2p-pubsub v0.5.4 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.4.0 // indirect
github.com/libp2p/go-libp2p-quic-transport v0.12.0 // indirect
github.com/libp2p/go-libp2p-record v0.1.3 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.2.3 // indirect
github.com/libp2p/go-libp2p-swarm v0.5.3 // indirect
github.com/libp2p/go-libp2p-tls v0.2.0 // indirect
github.com/libp2p/go-libp2p-transport-upgrader v0.4.6 // indirect
github.com/libp2p/go-libp2p-xor v0.0.0-20210714161855-5c005aca55db // indirect
github.com/libp2p/go-libp2p-yamux v0.5.4 // indirect
github.com/libp2p/go-maddr-filter v0.1.0 // indirect
github.com/libp2p/go-mplex v0.3.0 // indirect
github.com/libp2p/go-msgio v0.0.6 // indirect
github.com/libp2p/go-nat v0.0.5 // indirect
github.com/libp2p/go-netroute v0.1.6 // indirect
github.com/libp2p/go-openssl v0.0.7 // indirect
github.com/libp2p/go-reuseport v0.0.2 // indirect
github.com/libp2p/go-reuseport-transport v0.0.5 // indirect
github.com/libp2p/go-sockaddr v0.1.1 // indirect
github.com/libp2p/go-stream-muxer-multistream v0.3.0 // indirect
github.com/libp2p/go-tcp-transport v0.2.8 // indirect
github.com/libp2p/go-ws-transport v0.5.0 // indirect
github.com/libp2p/go-yamux/v2 v2.2.0 // indirect
github.com/libp2p/zeroconf/v2 v2.0.0 // indirect
github.com/lucas-clemente/quic-go v0.26.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/miekg/dns v1.1.58 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/miekg/dns v1.1.43 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multiaddr v0.12.2 // indirect
github.com/multiformats/go-base32 v0.0.3 // indirect
github.com/multiformats/go-base36 v0.1.0 // indirect
github.com/multiformats/go-multiaddr v0.4.0 // indirect
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multibase v0.2.0 // indirect
github.com/multiformats/go-multicodec v0.9.0 // indirect
github.com/multiformats/go-multistream v0.5.0 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/multiformats/go-multibase v0.0.3 // indirect
github.com/multiformats/go-multicodec v0.3.0 // indirect
github.com/multiformats/go-multistream v0.2.2 // indirect
github.com/multiformats/go-varint v0.0.6 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opencontainers/runtime-spec v1.2.0 // indirect
github.com/openrelayxyz/plugeth-utils v1.5.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
github.com/pganalyze/pg_query_go/v4 v4.2.1 // indirect
github.com/pion/datachannel v1.5.5 // indirect
github.com/pion/dtls/v2 v2.2.8 // indirect
github.com/pion/ice/v2 v2.3.11 // indirect
github.com/pion/interceptor v0.1.25 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns v0.0.9 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.13 // indirect
github.com/pion/rtp v1.8.3 // indirect
github.com/pion/sctp v1.8.9 // indirect
github.com/pion/sdp/v3 v3.0.6 // indirect
github.com/pion/srtp/v2 v2.0.18 // indirect
github.com/pion/stun v0.6.1 // indirect
github.com/pion/transport/v2 v2.2.4 // indirect
github.com/pion/turn/v2 v2.1.4 // indirect
github.com/pion/webrtc/v3 v3.2.23 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_golang v1.18.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.47.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/quic-go v0.41.0 // indirect
github.com/quic-go/webtransport-go v0.6.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/samber/lo v1.39.0 // indirect
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.30.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/prometheus/tsdb v0.7.1 // indirect
github.com/segmentio/fasthash v1.0.3 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/stretchr/testify v1.9.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/supranational/blst v0.3.11 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/thoas/go-funk v0.9.3 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
github.com/tklauser/go-sysconf v0.3.5 // indirect
github.com/tklauser/numcpus v0.2.2 // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20240109153615-66e95c3e8a87 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 // indirect
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.25.0 // indirect
go.opentelemetry.io/otel/metric v1.25.0 // indirect
go.opentelemetry.io/otel/trace v1.25.0 // indirect
go.uber.org/dig v1.17.1 // indirect
go.uber.org/fx v1.20.1 // indirect
go.uber.org/mock v0.4.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.20.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
gonum.org/v1/gonum v0.14.0 // indirect
google.golang.org/protobuf v1.32.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.2.2 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/otel v0.20.0 // indirect
go.opentelemetry.io/otel/metric v0.20.0 // indirect
go.opentelemetry.io/otel/trace v0.20.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/dig v1.10.0 // indirect
go.uber.org/fx v1.13.1 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.19.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
lukechampine.com/blake3 v1.1.6 // indirect
)

2164
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -18,7 +18,7 @@ package main
import (
"github.com/sirupsen/logrus"
"github.com/cerc-io/eth-ipfs-state-validator/v5/cmd"
"github.com/vulcanize/eth-ipfs-state-validator/v3/cmd"
)
func main() {

View File

@ -18,8 +18,6 @@ package validator
import (
"fmt"
"os"
"strconv"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
@ -34,6 +32,13 @@ const (
DATABASE_PASSWORD = "DATABASE_PASSWORD"
)
// NewDB returns a new sqlx.DB from config/cli/env variables
func NewDB() (*sqlx.DB, error) {
c := Config{}
c.Init()
return sqlx.Connect("postgres", c.ConnString())
}
type Config struct {
Hostname string
Name string
@ -42,13 +47,6 @@ type Config struct {
Port int
}
// NewDB returns a new sqlx.DB from config/cli/env variables
func NewDB() (*sqlx.DB, error) {
c := Config{}
LoadViper(&c)
return sqlx.Connect("postgres", c.ConnString())
}
func (c *Config) ConnString() string {
if len(c.User) > 0 && len(c.Password) > 0 {
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
@ -61,30 +59,7 @@ func (c *Config) ConnString() string {
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.Name)
}
func LoadEnv(c *Config) error {
if val := os.Getenv(DATABASE_NAME); val != "" {
c.Name = val
}
if val := os.Getenv(DATABASE_HOSTNAME); val != "" {
c.Hostname = val
}
if val := os.Getenv(DATABASE_PORT); val != "" {
port, err := strconv.Atoi(val)
if err != nil {
return err
}
c.Port = port
}
if val := os.Getenv(DATABASE_USER); val != "" {
c.User = val
}
if val := os.Getenv(DATABASE_PASSWORD); val != "" {
c.Password = val
}
return nil
}
func LoadViper(c *Config) {
func (c *Config) Init() {
viper.BindEnv("database.name", DATABASE_NAME)
viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
viper.BindEnv("database.port", DATABASE_PORT)

View File

@ -19,7 +19,7 @@ package validator_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

View File

@ -1,9 +0,0 @@
package validator
type TraversalType = string
const (
fullTraversal = "full"
stateTraversal = "state"
storageTraversal = "storage"
)

View File

@ -19,11 +19,42 @@ package validator
import (
"context"
"github.com/ipfs/boxo/blockservice"
"github.com/ipfs/kubo/core"
"github.com/ipfs/kubo/repo/fsrepo"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/repo/fsrepo"
"github.com/jmoiron/sqlx"
)
// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx
func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte) (string, error) {
c, err := RawdataToCid(codec, raw, mh)
if err != nil {
return "", err
}
dbKey := dshelp.MultihashToDsKey(c.Hash())
prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
_, err = tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw)
return c.String(), err
}
// RawdataToCid takes the desired codec, multihash type, and a slice of bytes
// and returns the proper cid of the object.
func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) {
c, err := cid.Prefix{
Codec: codec,
Version: 1,
MhType: multiHash,
MhLength: -1,
}.Sum(rawdata)
if err != nil {
return cid.Cid{}, err
}
return c, nil
}
// InitIPFSBlockService is used to configure and return a BlockService using an ipfs repo path (e.g. ~/.ipfs)
func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
r, openErr := fsrepo.Open(ipfsPath)
@ -41,3 +72,9 @@ func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
}
return ipfsNode.Blocks, nil
}
// ResetTestDB drops all rows in the test db public.blocks table
func ResetTestDB(db *sqlx.DB) error {
_, err := db.Exec("DELETE FROM public.blocks")
return err
}

View File

@ -1,39 +0,0 @@
package validator_test
import (
"github.com/ipfs/go-cid"
"github.com/jmoiron/sqlx"
)
// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx
func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte, blockNumber uint64) error {
c, err := RawdataToCid(codec, raw, mh)
if err != nil {
return err
}
_, err = tx.Exec(
`INSERT INTO ipld.blocks (key, data, block_number) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`,
c.String(), raw, blockNumber)
return err
}
// RawdataToCid takes the desired codec, multihash type, and a slice of bytes
// and returns the proper cid of the object.
func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) {
c, err := cid.Prefix{
Codec: codec,
Version: 1,
MhType: multiHash,
MhLength: -1,
}.Sum(rawdata)
if err != nil {
return cid.Cid{}, err
}
return c, nil
}
// ResetTestDB truncates all used tables from the test DB
func ResetTestDB(db *sqlx.DB) error {
_, err := db.Exec("TRUNCATE ipld.blocks")
return err
}

View File

@ -17,67 +17,31 @@
package validator
import (
"bytes"
"context"
"fmt"
"os"
"os/signal"
"syscall"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/boxo/blockservice"
"github.com/ethereum/go-ethereum/trie"
"github.com/ipfs/go-blockservice"
"github.com/jmoiron/sqlx"
"github.com/mailgun/groupcache/v2"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
iterutils "github.com/cerc-io/eth-iterator-utils"
"github.com/cerc-io/eth-iterator-utils/tracker"
ipfsethdb "github.com/cerc-io/ipfs-ethdb/v5"
pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0"
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/state"
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/triedb"
ipfsethdb "github.com/vulcanize/ipfs-ethdb/v3"
pgipfsethdb "github.com/vulcanize/ipfs-ethdb/v3/postgres"
)
// Validator is used for validating Ethereum state and storage tries on PG-IPFS
type Validator struct {
kvs ethdb.KeyValueStore
trieDB *triedb.Database
trieDB *trie.Database
stateDatabase state.Database
db *pgipfsethdb.Database
params Params
}
type Params struct {
Workers uint
RecoveryFormat string // %s substituted with traversal type
}
var (
DefaultRecoveryFormat = "./recover_validate_%s"
emptyCodeHash = crypto.Keccak256(nil)
)
type KVSWithAncient struct {
kvs ethdb.KeyValueStore
ethdb.Database
}
func NewKVSDatabaseWithAncient(kvs ethdb.KeyValueStore) ethdb.Database {
return &KVSWithAncient{
kvs: kvs,
}
}
// NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database
func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator {
func NewPGIPFSValidator(db *sqlx.DB) *Validator {
kvs := pgipfsethdb.NewKeyValueStore(db, pgipfsethdb.CacheConfig{
Name: "kv",
Size: 16 * 1000 * 1000, // 16MB
@ -90,13 +54,11 @@ func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator {
ExpiryDuration: time.Hour * 8, // 8 hours
})
normalizeParams(&par)
return &Validator{
kvs: kvs,
trieDB: triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil),
trieDB: trie.NewDatabase(kvs),
stateDatabase: state.NewDatabase(database),
db: database.(*pgipfsethdb.Database),
params: par,
}
}
@ -105,15 +67,13 @@ func (v *Validator) GetCacheStats() groupcache.Stats {
}
// NewIPFSValidator returns a new trie validator ontop of an IPFS blockservice
func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
func NewIPFSValidator(bs blockservice.BlockService) *Validator {
kvs := ipfsethdb.NewKeyValueStore(bs)
database := ipfsethdb.NewDatabase(bs)
normalizeParams(&par)
return &Validator{
kvs: kvs,
trieDB: triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil),
trieDB: trie.NewDatabase(kvs),
stateDatabase: state.NewDatabase(database),
params: par,
}
}
@ -123,30 +83,26 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
return &Validator{
kvs: kvs,
trieDB: triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil),
trieDB: trie.NewDatabase(kvs),
stateDatabase: state.NewDatabase(database),
}
}
// Ensure params are valid
func normalizeParams(p *Params) {
if p.Workers == 0 {
p.Workers = 1
}
if len(p.RecoveryFormat) == 0 {
p.RecoveryFormat = DefaultRecoveryFormat
}
}
// ValidateTrie returns an error if the state and storage tries for the provided state root cannot be confirmed as complete
// This does consider child storage tries
func (v *Validator) ValidateTrie(stateRoot common.Hash) error {
t, err := v.stateDatabase.OpenTrie(stateRoot)
// Generate the state.NodeIterator for this root
stateDB, err := state.New(stateRoot, v.stateDatabase, nil)
if err != nil {
return err
}
iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, true) }
return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, fullTraversal), v.params.Workers, iterate)
it := state.NewNodeIterator(stateDB)
for it.Next() {
// iterate through entire state trie and descendent storage tries
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
// if we are able to iterate through the entire trie without error then the trie is complete
}
return it.Error
}
// ValidateStateTrie returns an error if the state trie for the provided state root cannot be confirmed as complete
@ -157,21 +113,30 @@ func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error {
if err != nil {
return err
}
iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, false) }
return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, stateTraversal), v.params.Workers, iterate)
it := t.NodeIterator(nil)
for it.Next(true) {
// iterate through entire state trie
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
// if we are able to iterate through the entire trie without error then the trie is complete
}
return it.Error()
}
// ValidateStorageTrie returns an error if the storage trie for the provided storage root and contract address cannot be confirmed as complete
func (v *Validator) ValidateStorageTrie(stateRoot common.Hash, address common.Address, storageRoot common.Hash) error {
func (v *Validator) ValidateStorageTrie(address common.Address, storageRoot common.Hash) error {
// Generate the state.NodeIterator for this root
addrHash := crypto.Keccak256Hash(address.Bytes())
// Note: the last argument is the redundant state trie, but will be needed for Verkle tries
storage, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot, nil)
t, err := v.stateDatabase.OpenStorageTrie(addrHash, storageRoot)
if err != nil {
return err
}
iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, false) }
return iterateTracked(storage, fmt.Sprintf(v.params.RecoveryFormat, storageTraversal), v.params.Workers, iterate)
it := t.NodeIterator(nil)
for it.Next(true) {
// iterate through entire storage trie
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
// if we are able to iterate through the entire trie without error then the trie is complete
}
return it.Error()
}
// Close implements io.Closer
@ -181,111 +146,3 @@ func (v *Validator) Close() error {
groupcache.DeregisterGroup("db")
return nil
}
// Traverses one iterator fully
// If storage = true, also traverse storage tries for each leaf.
func (v *Validator) iterate(ctx context.Context, stateRoot common.Hash, it trie.NodeIterator, storage bool) error {
// Iterate through entire state trie. it.Next() will return false when we have
// either completed iteration of the entire trie or run into an error (e.g. a
// missing node). If we are able to iterate through the entire trie without error
// then the trie is complete.
for it.Next(true) {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
// This block adapted from geth - core/state/iterator.go
// If storage is not requested, or the state trie node is an internal entry, skip
if !storage || !it.Leaf() {
continue
}
// Otherwise we've reached an account node, initiate data iteration
var account types.StateAccount
if err := rlp.Decode(bytes.NewReader(it.LeafBlob()), &account); err != nil {
return err
}
// Note: the last argument is the redundant state trie, but will be needed for Verkle tries
dataTrie, err := v.stateDatabase.OpenStorageTrie(stateRoot, common.BytesToHash(it.LeafKey()), account.Root, nil)
if err != nil {
return err
}
dataIt, err := dataTrie.NodeIterator(nil)
if err != nil {
return err
}
if !bytes.Equal(account.CodeHash, emptyCodeHash) {
_, err := v.stateDatabase.ContractCode(common.Address{}, common.BytesToHash(account.CodeHash))
if err != nil {
return fmt.Errorf("code hash %x: %w (path %x)", account.CodeHash, err, iterutils.HexToKeyBytes(it.Path()))
}
}
for dataIt.Next(true) {
}
if dataIt.Error() != nil {
return fmt.Errorf("data iterator error (path %x): %w", iterutils.HexToKeyBytes(dataIt.Path()), dataIt.Error())
}
}
return it.Error()
}
// Traverses each iterator in a separate goroutine.
// Dumps to a recovery file on failure or interrupt.
func iterateTracked(
tree state.Trie,
recoveryFile string,
iterCount uint,
fn func(context.Context, trie.NodeIterator) error,
) error {
tracker := tracker.New(recoveryFile, iterCount)
halt := func() {
if err := tracker.CloseAndSave(); err != nil {
log.Errorf("failed to write recovery file: %v", err)
}
}
// attempt to restore from recovery file if it exists
iters, _, err := tracker.Restore(tree.NodeIterator)
if err != nil {
return err
}
if iterCount < uint(len(iters)) {
return fmt.Errorf("recovered too many iterators: got %d, expected %d", len(iters), iterCount)
}
if iters == nil { // nothing restored
iters, err = iterutils.SubtrieIterators(tree.NodeIterator, iterCount)
if err != nil {
return err
}
for i, it := range iters {
iters[i] = tracker.Tracked(it)
}
} else {
log.Debugf("restored %d iterators from: %s", len(iters), recoveryFile)
}
ctx, cancel := context.WithCancel(context.Background())
g, ctx := errgroup.WithContext(ctx)
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigChan
log.Errorf("Signal received (%v), stopping", sig)
cancel()
}()
defer halt()
for _, it := range iters {
func(it trie.NodeIterator) {
g.Go(func() error {
return fn(ctx, it)
})
}(it)
}
return g.Wait()
}

View File

@ -17,25 +17,24 @@
package validator_test
import (
"os"
"path/filepath"
"fmt"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/holiman/uint256"
cid "github.com/ipfs/go-cid/_rsrch/cidiface"
"github.com/jmoiron/sqlx"
"github.com/multiformats/go-multihash"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
validator "github.com/cerc-io/eth-ipfs-state-validator/v5/pkg"
validator "github.com/vulcanize/eth-ipfs-state-validator/v3/pkg"
pgipfsethdb "github.com/vulcanize/ipfs-ethdb/v3/postgres"
)
var (
blockNumber = uint64(1)
contractAddr = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
slot0StorageValue = common.Hex2Bytes("94703c4b2bd70c169f5717101caee543299fc946c7")
slot1StorageValue = common.Hex2Bytes("01")
@ -66,10 +65,9 @@ var (
mockCode = []byte{1, 2, 3, 4, 5}
codeHash = crypto.Keccak256Hash(mockCode)
codePath = common.Hex2Bytes("6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
contractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
Nonce: 1,
Balance: uint256.NewInt(0),
Balance: big.NewInt(0),
CodeHash: codeHash.Bytes(),
Root: crypto.Keccak256Hash(storageBranchRootNode),
})
@ -80,7 +78,7 @@ var (
minerAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
Nonce: 0,
Balance: uint256.NewInt(1000),
Balance: big.NewInt(1000),
CodeHash: nullCodeHash.Bytes(),
Root: emptyContractRoot,
})
@ -91,7 +89,7 @@ var (
account1, _ = rlp.EncodeToBytes(&types.StateAccount{
Nonce: 2,
Balance: uint256.NewInt(1000),
Balance: big.NewInt(1000),
CodeHash: nullCodeHash.Bytes(),
Root: emptyContractRoot,
})
@ -102,7 +100,7 @@ var (
account2, _ = rlp.EncodeToBytes(&types.StateAccount{
Nonce: 0,
Balance: uint256.NewInt(1000),
Balance: big.NewInt(1000),
CodeHash: nullCodeHash.Bytes(),
Root: emptyContractRoot,
})
@ -113,7 +111,7 @@ var (
bankAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
Nonce: 2,
Balance: uint256.NewInt(1000),
Balance: big.NewInt(1000),
CodeHash: nullCodeHash.Bytes(),
Root: emptyContractRoot,
})
@ -189,84 +187,63 @@ var (
storageBranchRootNode,
slot1StorageLeafNode,
}
missingStateNodePath = common.Hex2Bytes("0e")
missingStorageNodePath = common.Hex2Bytes("02")
)
var (
v *validator.Validator
db *sqlx.DB
err error
tmp string
config = validator.Config{
Hostname: "localhost",
Name: "cerc_testing",
User: "vdbm",
Password: "password",
Port: 8077,
}
)
var _ = Describe("PG-IPFS Validator", func() {
BeforeEach(func() {
err = validator.LoadEnv(&config)
db, err = pgipfsethdb.TestDB()
Expect(err).ToNot(HaveOccurred())
db, err = sqlx.Connect("postgres", config.ConnString())
Expect(err).ToNot(HaveOccurred())
tmp, err = os.MkdirTemp("", "test_validator")
Expect(err).ToNot(HaveOccurred())
params := validator.Params{Workers: 4, RecoveryFormat: filepath.Join(tmp, "recover_%s")}
v = validator.NewPGIPFSValidator(db, params)
v = validator.NewPGIPFSValidator(db)
})
AfterEach(func() {
os.RemoveAll(tmp)
v.Close()
db.Close()
})
Describe("ValidateTrie", func() {
AfterEach(func() {
err = ResetTestDB(db)
err = validator.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
It("Returns an error if the state root node is missing", func() {
It("Returns an error the state root node is missing", func() {
// we write code to ethdb, there should probably be an EthCode IPLD codec
// but there isn't, and we don't need one here since blockstore keys are mh-derived
loadTrie(missingRootStateNodes, trieStorageNodes, mockCode)
loadTrie(append(missingRootStateNodes, mockCode), trieStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
})
It("Returns an error if the storage root node is missing", func() {
loadTrie(trieStateNodes, missingRootStorageNodes, mockCode)
loadTrie(append(trieStateNodes, mockCode), missingRootStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
})
It("Returns an error if the state trie is missing node(s)", func() {
loadTrie(missingNodeStateNodes, trieStorageNodes, mockCode)
loadTrie(append(missingNodeStateNodes, mockCode), trieStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
Expect(err.Error()).To(ContainSubstring("path %x", missingStateNodePath))
})
It("Returns an error if the storage trie is missing node(s)", func() {
loadTrie(trieStateNodes, missingNodeStorageNodes, mockCode)
loadTrie(append(trieStateNodes, mockCode), missingNodeStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
Expect(err.Error()).To(ContainSubstring("path %x", missingStorageNodePath))
})
It("Returns an error if contract code is missing", func() {
loadTrie(trieStateNodes, trieStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("%x", codeHash))
Expect(err.Error()).To(ContainSubstring("%x", codePath))
subStr := fmt.Sprintf("code %s: not found", codeHash.Hex()[2:])
Expect(err.Error()).To(ContainSubstring(subStr))
})
It("Returns no error if the entire state (state trie and storage tries) can be validated", func() {
loadTrie(trieStateNodes, trieStorageNodes, mockCode)
loadTrie(append(trieStateNodes, mockCode), trieStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).ToNot(HaveOccurred())
})
@ -274,7 +251,7 @@ var _ = Describe("PG-IPFS Validator", func() {
Describe("ValidateStateTrie", func() {
AfterEach(func() {
err = ResetTestDB(db)
err = validator.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
It("Returns an error the state root node is missing", func() {
@ -288,7 +265,6 @@ var _ = Describe("PG-IPFS Validator", func() {
err = v.ValidateStateTrie(stateRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
Expect(err.Error()).To(ContainSubstring("path %x", missingStateNodePath))
})
It("Returns no error if the entire state trie can be validated", func() {
loadTrie(trieStateNodes, nil)
@ -299,43 +275,38 @@ var _ = Describe("PG-IPFS Validator", func() {
Describe("ValidateStorageTrie", func() {
AfterEach(func() {
err = ResetTestDB(db)
err = validator.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
It("Returns an error if the storage root node is missing", func() {
It("Returns an error the storage root node is missing", func() {
loadTrie(nil, missingRootStorageNodes)
err = v.ValidateStorageTrie(common.Hash{}, contractAddr, storageRoot)
err = v.ValidateStorageTrie(contractAddr, storageRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
})
It("Returns an error if the entire storage trie cannot be validated", func() {
loadTrie(nil, missingNodeStorageNodes)
err = v.ValidateStorageTrie(common.Hash{}, contractAddr, storageRoot)
err = v.ValidateStorageTrie(contractAddr, storageRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
Expect(err.Error()).To(ContainSubstring("path %x", missingStorageNodePath))
})
It("Returns no error if the entire storage trie can be validated", func() {
loadTrie(nil, trieStorageNodes)
err = v.ValidateStorageTrie(common.Hash{}, contractAddr, storageRoot)
err = v.ValidateStorageTrie(contractAddr, storageRoot)
Expect(err).ToNot(HaveOccurred())
})
})
})
func loadTrie(stateNodes, storageNodes [][]byte, contractCode ...[]byte) {
func loadTrie(stateNodes, storageNodes [][]byte) {
tx, err := db.Beginx()
Expect(err).ToNot(HaveOccurred())
for _, node := range stateNodes {
err := PublishRaw(tx, cid.EthStateTrie, multihash.KECCAK_256, node, blockNumber)
_, err := validator.PublishRaw(tx, cid.EthStateTrie, multihash.KECCAK_256, node)
Expect(err).ToNot(HaveOccurred())
}
for _, node := range storageNodes {
err := PublishRaw(tx, cid.EthStorageTrie, multihash.KECCAK_256, node, blockNumber)
Expect(err).ToNot(HaveOccurred())
}
for _, code := range contractCode {
err := PublishRaw(tx, cid.Raw, multihash.KECCAK_256, code, blockNumber)
_, err := validator.PublishRaw(tx, cid.EthStorageTrie, multihash.KECCAK_256, node)
Expect(err).ToNot(HaveOccurred())
}
err = tx.Commit()

View File

@ -1,28 +0,0 @@
# Containers to run backing DB for testing
services:
migrations:
restart: on-failure
depends_on:
- ipld-eth-db
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha
environment:
DATABASE_USER: "vdbm"
DATABASE_NAME: "cerc_testing"
DATABASE_PASSWORD: "password"
DATABASE_HOSTNAME: "ipld-eth-db"
DATABASE_PORT: 5432
ipld-eth-db:
container_name: test-ipld-eth-db
image: timescale/timescaledb:latest-pg14
restart: always
command: ["postgres", "-c", "log_statement=all"]
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "cerc_testing"
POSTGRES_PASSWORD: "password"
ports:
- "127.0.0.1:8077:5432"
volumes:
- ./statediff/indexer/database/file:/file_indexer