Updates for Cancun fork (#7)

- Bumps Geth to v1.14
- Adds support for pebbledb
- Drops leveldb-ethdb-rpc

Reviewed-on: #7
This commit is contained in:
Roy Crihfield 2024-08-05 13:08:26 +00:00
parent be105a4744
commit cdacea3aae
20 changed files with 286 additions and 922 deletions

View File

@ -1 +1,4 @@
.git
Dockerfile
.git
.gitea
test

View File

@ -9,9 +9,8 @@ on:
- ci-test
workflow_call:
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
ETH_TESTING_REF: v0.5.2
jobs:
integration-tests:
@ -23,17 +22,8 @@ jobs:
with:
go-version-file: go.mod
check-latest: true
- name: Run dockerd
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: Run DB container
run: docker compose -f test/compose.yml up --wait
- name: Configure Gitea access
env:
TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
run: |
git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"
- name: Build package
run: go build .
@ -42,16 +32,15 @@ jobs:
with:
repository: cerc-io/eth-testing
path: ./fixtures
ref: v0.3.1
ref: ${{ env.ETH_TESTING_REF }}
# Run a sanity test against the fixture data
# Complete integration tests are TODO
- name: Run basic integration test
env:
DATABASE_TYPE: postgres
LEVELDB_PATH: ./fixtures/chaindata/_data/small2
LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
ETHDB_PATH: ./fixtures/chains/data/premerge2/geth/chaindata
ETH_CHAIN_CONFIG: ./fixtures/chains/data/premerge2/genesis.config.json
ETH_GENESIS_BLOCK: 0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614
LOG_FILE: ./server-log
ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
timeout-minutes: 20
run: |
./eth-statediff-service --config ./test/ci-config.toml serve &
@ -85,17 +74,6 @@ jobs:
with:
go-version-file: ./eth-statediff-service/go.mod
check-latest: true
- name: Install test fixtures
uses: actions/checkout@v3
with:
repository: cerc-io/eth-testing
path: ./fixtures
ref: v0.3.1
- name: Configure Gitea access
env:
TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
run: |
git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf https://git.vdb.to/
- name: Build current version
working-directory: ./eth-statediff-service
run: go build -o ../service-current .
@ -109,26 +87,28 @@ jobs:
working-directory: ./eth-statediff-service-canonical
run: go build -o ../service-canonical .
- name: Run dockerd
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: Install test fixtures
uses: actions/checkout@v3
with:
repository: cerc-io/eth-testing
path: ./fixtures
ref: ${{ env.ETH_TESTING_REF }}
- name: Run DB container
working-directory: ./eth-statediff-service
run: docker compose -f test/compose.yml up --wait
- name: Compare statediff output
timeout-minutes: 10
env:
LEVELDB_PATH: ./fixtures/chaindata/_data/small2
LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
ETHDB_PATH: ./fixtures/chains/data/premerge2/geth/chaindata
ETHDB_ANCIENT: ./fixtures/chains/data/premerge2/geth/chaindata/ancient
ETH_CHAIN_CONFIG: ./fixtures/chains/data/premerge2/genesis.config.json
ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
ETH_CHAIN_CONFIG: ./eth-statediff-service/test/ci-chain.json
run: |
until
ready_query='select max(version_id) from goose_db_version;'
version=$(docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
psql -tA cerc_testing -U vdbm -c "$ready_query")
[[ "$version" -ge 18 ]]
[[ "$version" -ge 21 ]]
do sleep 1; done
./eth-statediff-service/scripts/compare-statediffs.sh \

View File

@ -1,4 +1,4 @@
FROM golang:1.19-alpine as builder
FROM golang:1.21-alpine as builder
RUN apk add --no-cache git gcc musl-dev binutils-gold
# DEBUG

View File

@ -2,7 +2,7 @@
[![Go Report Card](https://goreportcard.com/badge/github.com/cerc-io/eth-statediff-service)](https://goreportcard.com/report/github.com/cerc-io/eth-statediff-service)
A standalone statediffing service which runs directly on top of a `go-ethereum` LevelDB instance.
A standalone statediffing service which runs directly on top of a `go-ethereum` database instance.
This service can serve historical state data over the same rpc interface as
[statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.
@ -19,7 +19,7 @@ go build .
See [./environments/example.toml](./environments/example.toml) for an annotated example config file.
> **Note:** previous versions of this service used different variable names. To update, change the following:
> * `LVLDB_MODE`, `LVLDB_PATH`, `LVLDB_ANCIENT`, `LVLDB_URL` => `LEVELDB_*`
> * `LVLDB_*`, `LEVELDB_*` => `ETHDB_*`
> * `LOG_FILE_PATH` => `LOG_FILE`
### Local Setup
@ -208,9 +208,9 @@ ranges and params in the `prerun` section of the config.
### Stats
The binary includes a `stats` command which reports stats for the offline or remote levelDB.
The binary includes a `stats` command which reports stats for the offline DB.
At this time, the only stat supported is to return the latest/highest block height and hash found the levelDB, this is
useful for determining what the upper limit is for a standalone statediffing process on a given levelDB.
At this time, the only stat supported is to return the latest/highest block height and hash found in the EthDB. This is
useful for determining what the upper limit is for a standalone statediffing process using a given EthDB.
`./eth-statediff-service stats --config={path to toml config file}`

View File

@ -30,10 +30,9 @@ const (
DB_CACHE_SIZE_MB = "DB_CACHE_SIZE_MB"
TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
LEVELDB_MODE = "LEVELDB_MODE"
LEVELDB_PATH = "LEVELDB_PATH"
LEVELDB_ANCIENT = "LEVELDB_ANCIENT"
LEVELDB_URL = "LEVELDB_URL"
// ETHDB_ENGINE = "ETHDB_ENGINE"
ETHDB_PATH = "ETHDB_PATH"
ETHDB_ANCIENT = "ETHDB_ANCIENT"
STATEDIFF_PRERUN = "STATEDIFF_PRERUN"
STATEDIFF_TRIE_WORKERS = "STATEDIFF_TRIE_WORKERS"
@ -119,10 +118,9 @@ func init() {
viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)
viper.BindEnv("leveldb.mode", LEVELDB_MODE)
viper.BindEnv("leveldb.path", LEVELDB_PATH)
viper.BindEnv("leveldb.ancient", LEVELDB_ANCIENT)
viper.BindEnv("leveldb.url", LEVELDB_URL)
// viper.BindEnv("ethdb.engine", ETHDB_ENGINE)
viper.BindEnv("ethdb.path", ETHDB_PATH)
viper.BindEnv("ethdb.ancient", ETHDB_ANCIENT)
viper.BindEnv("prom.metrics", PROM_METRICS)
viper.BindEnv("prom.http", PROM_HTTP)

View File

@ -115,10 +115,9 @@ func init() {
rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(),
"log level (trace, debug, info, warn, error, fatal, panic")
rootCmd.PersistentFlags().String("leveldb-mode", "local", "LevelDB access mode (local, remote)")
rootCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
// rootCmd.PersistentFlags().String("ethdb-engine", "local", "Ethdb engine type (leveldb, pebble)")
rootCmd.PersistentFlags().String("ethdb-path", "", "path to primary datastore")
rootCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
rootCmd.PersistentFlags().String("leveldb-url", "", "url to primary leveldb-ethdb-rpc server")
rootCmd.PersistentFlags().Bool("prerun", false, "turn on prerun of toml configured ranges")
rootCmd.PersistentFlags().Int("service-workers", 1, "number of range requests to process concurrently")
@ -178,10 +177,10 @@ func init() {
viper.BindPFlag("statediff.trieWorkers", rootCmd.PersistentFlags().Lookup("trie-workers"))
viper.BindPFlag("statediff.workerQueueSize", rootCmd.PersistentFlags().Lookup("worker-queue-size"))
viper.BindPFlag("leveldb.mode", rootCmd.PersistentFlags().Lookup("leveldb-mode"))
viper.BindPFlag("leveldb.path", rootCmd.PersistentFlags().Lookup("leveldb-path"))
viper.BindPFlag("leveldb.ancient", rootCmd.PersistentFlags().Lookup("ancient-path"))
viper.BindPFlag("leveldb.url", rootCmd.PersistentFlags().Lookup("leveldb-url"))
// viper.BindPFlag("ethdb.engine", rootCmd.PersistentFlags().Lookup("ethdb-engine"))
viper.BindPFlag("ethdb.path", rootCmd.PersistentFlags().Lookup("ethdb-path"))
viper.BindPFlag("ethdb.ancient", rootCmd.PersistentFlags().Lookup("ancient-path"))
viper.BindPFlag("ethdb.url", rootCmd.PersistentFlags().Lookup("ethdb-url"))
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@ -235,7 +234,7 @@ func initConfig() {
if cfgFile != "" {
viper.SetConfigFile(cfgFile)
if err := viper.ReadInConfig(); err == nil {
log.Printf("Using config file: %s", viper.ConfigFileUsed())
log.Infof("Using config file: %s", viper.ConfigFileUsed())
} else {
log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
}

View File

@ -35,8 +35,8 @@ import (
// serveCmd represents the serve command
var serveCmd = &cobra.Command{
Use: "serve",
Short: "Stand up a standalone statediffing RPC service on top of LevelDB",
Long: `Usage
Short: "Standalone statediffing RPC service on top of an Ethereum database",
Long: `Usage:
./eth-statediff-service serve --config={path to toml config file}`,
Run: func(cmd *cobra.Command, args []string) {
@ -63,7 +63,7 @@ func serve() {
logWithCommand.Debug("Running eth-statediff-service serve command")
logWithCommand.Debugf("Parallelism: %d", maxParallelism())
reader, chainConf, nodeInfo := instantiateLevelDBReader()
reader, chainConf, nodeInfo := createReader()
reportLatestBlock(reader)

View File

@ -23,7 +23,7 @@ import (
// statsCmd represents the serve command
var statsCmd = &cobra.Command{
Use: "stats",
Short: "Report stats for cold levelDB",
Short: "Report stats for cold DB",
Long: `Usage
./eth-statediff-service stats --config={path to toml config file}`,
@ -41,6 +41,6 @@ func init() {
func stats() {
logWithCommand.Info("Running eth-statediff-service stats command")
reader, _, _ := instantiateLevelDBReader()
reader, _, _ := createReader()
reportLatestBlock(reader)
}

View File

@ -10,7 +10,8 @@ import (
"github.com/cerc-io/plugeth-statediff/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/triedb"
"github.com/ethereum/go-ethereum/triedb/hashdb"
"github.com/spf13/viper"
pkg "github.com/cerc-io/eth-statediff-service/pkg"
@ -86,24 +87,18 @@ func setupPreRunRanges() []pkg.RangeRequest {
return blockRanges
}
func instantiateLevelDBReader() (pkg.Reader, *params.ChainConfig, node.Info) {
func createReader() (pkg.Reader, *params.ChainConfig, node.Info) {
// load some necessary params
logWithCommand.Debug("Loading statediff service parameters")
mode := viper.GetString("leveldb.mode")
path := viper.GetString("leveldb.path")
ancientPath := viper.GetString("leveldb.ancient")
url := viper.GetString("leveldb.url")
path := viper.GetString("ethdb.path")
ancientPath := viper.GetString("ethdb.ancient")
url := viper.GetString("ethdb.url")
if mode == "local" {
if path == "" || ancientPath == "" {
logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
}
} else if mode == "remote" {
if url == "" {
logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
}
} else {
logWithCommand.Fatal("Invalid mode provided for LevelDB access")
if path == "" {
logWithCommand.Fatal("Require a valid Ethereum chain data path")
}
if ancientPath == "" {
ancientPath = path + "/ancient"
}
nodeInfo := getEthNodeInfo()
@ -114,24 +109,24 @@ func instantiateLevelDBReader() (pkg.Reader, *params.ChainConfig, node.Info) {
logWithCommand.Fatalf("Unable to instantiate chain config: %s", err)
}
// create LevelDB reader
logWithCommand.Debug("Creating LevelDB reader")
readerConf := pkg.LvLDBReaderConfig{
TrieConfig: &trie.Config{
Cache: viper.GetInt("cache.trie"),
Journal: "",
logWithCommand.Debug("Creating DB reader")
readerConf := pkg.EthDBReaderConfig{
TrieConfig: &triedb.Config{
Preimages: false,
IsVerkle: false,
HashDB: &hashdb.Config{
CleanCacheSize: viper.GetInt("cache.trie"),
},
},
ChainConfig: chainConf,
Mode: mode,
Path: path,
AncientPath: ancientPath,
Url: url,
DBCacheSize: viper.GetInt("cache.database"),
}
reader, err := pkg.NewLvlDBReader(readerConf)
reader, err := pkg.NewEthDBReader(readerConf)
if err != nil {
logWithCommand.Fatalf("Unable to instantiate levelDB reader: %s", err)
logWithCommand.Fatalf("Unable to instantiate DB reader: %s", err)
}
return reader, chainConf, nodeInfo
}
@ -143,10 +138,10 @@ func reportLatestBlock(reader pkg.Reader) {
logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
}
if header.Number == nil {
logWithCommand.Fatal("Latest header found in levelDB has a nil block height")
logWithCommand.Fatal("Latest header found in DB has a nil block height")
}
logWithCommand.
WithField("height", header.Number).
WithField("hash", header.Hash()).
Info("Latest block found in levelDB")
Info("Latest block found in DB")
}

View File

@ -1,8 +1,6 @@
[leveldb]
mode = "local" # LEVELDB_MODE
path = "/app/geth-rw/chaindata" # LEVELDB_PATH
ancient = "/app/geth-rw/chaindata/ancient" # LEVELDB_ANCIENT
url = "http://127.0.0.1:8082/" # LEVELDB_URL
[ethdb]
path = "/app/geth-rw/chaindata" # ETHDB_PATH
ancient = "/app/geth-rw/chaindata/ancient" # ETHDB_ANCIENT
[server]
ipcPath = ".ipc" # SERVICE_IPC_PATH

View File

@ -1,13 +1,10 @@
[leveldb]
# LevelDB access mode <local | remote>
mode = "local" # LEVELDB_MODE
[ethdb]
# # access mode <local | remote>
# mode = "local" # ETHDB_MODE
# LevelDB paths (local mode)
path = "/Users/user/Library/Ethereum/geth/chaindata" # LEVELDB_PATH
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT
# URL for leveldb-ethdb-rpc endpoint (remote mode)
url = "http://127.0.0.1:8082/" # LEVELDB_URL
# Ethdb paths (local mode)
path = "/Users/user/Library/Ethereum/geth/chaindata" # ETHDB_PATH
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # ETHDB_ANCIENT
[server]
ipcPath = ".ipc" # SERVICE_IPC_PATH

88
go.mod
View File

@ -1,56 +1,63 @@
module github.com/cerc-io/eth-statediff-service
go 1.19
go 1.21
require (
github.com/cerc-io/leveldb-ethdb-rpc v1.1.13
github.com/cerc-io/plugeth-statediff v0.0.0-00010101000000-000000000000
github.com/ethereum/go-ethereum v1.12.0
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/cerc-io/plugeth-statediff v0.3.0
github.com/ethereum/go-ethereum v1.14.5
github.com/prometheus/client_golang v1.16.0
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.3.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.5.0
github.com/spf13/viper v1.10.1
)
require (
github.com/DataDog/zstd v1.5.5 // indirect
github.com/VictoriaMetrics/fastcache v1.12.1 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.10.0 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
github.com/cerc-io/eth-iterator-utils v1.2.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cockroachdb/errors v1.10.0 // indirect
github.com/cerc-io/eth-iterator-utils v0.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cockroachdb/errors v1.11.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/pebble v0.0.0-20230720154706-692f3b61a3c4 // indirect
github.com/cockroachdb/pebble v1.1.0 // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230613231145-182959a1fad6 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/consensys/bavard v0.1.13 // indirect
github.com/consensys/gnark-crypto v0.12.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/deckarep/golang-set/v2 v2.3.0 // indirect
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/deepmap/oapi-codegen v1.8.2 // indirect
github.com/edsrzf/mmap-go v1.0.0 // indirect
github.com/fjl/memsize v0.0.1 // indirect
github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 // indirect
github.com/ferranbt/fastssz v0.1.2 // indirect
github.com/fjl/memsize v0.0.2 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
github.com/georgysavva/scany v0.2.9 // indirect
github.com/getsentry/sentry-go v0.22.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-stack/stack v1.8.1 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/graph-gophers/graphql-go v1.3.0 // indirect
github.com/hashicorp/go-bexpr v0.1.12 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
github.com/holiman/uint256 v1.2.3 // indirect
github.com/huin/goupnp v1.2.0 // indirect
github.com/holiman/uint256 v1.2.4 // indirect
github.com/huin/goupnp v1.3.0 // indirect
github.com/inconshreveable/log15 v2.16.0+incompatible // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
@ -67,6 +74,7 @@ require (
github.com/jackc/pgx/v4 v4.13.0 // indirect
github.com/jackc/puddle v1.1.3 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jmoiron/sqlx v1.2.0 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/kr/pretty v0.3.1 // indirect
@ -74,12 +82,13 @@ require (
github.com/lib/pq v1.10.9 // indirect
github.com/magiconair/properties v1.8.5 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/pointerstructure v1.2.1 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
@ -87,7 +96,7 @@ require (
github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/openrelayxyz/plugeth-utils v1.2.0 // indirect
github.com/openrelayxyz/plugeth-utils v1.5.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
@ -98,7 +107,7 @@ require (
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/rs/cors v1.9.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
@ -110,37 +119,30 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/status-im/keycard-go v0.2.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/stretchr/testify v1.8.2 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/supranational/blst v0.3.11 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
github.com/thoas/go-funk v0.9.3 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
github.com/urfave/cli/v2 v2.25.7 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.2.1 // indirect
)
replace (
github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.1.2
github.com/cerc-io/eth-testing => git.vdb.to/cerc-io/eth-testing v0.3.1
github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.1.4
github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1
github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46
rsc.io/tmplfunc v0.0.3 // indirect
)

797
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -28,7 +28,7 @@ const APIName = "statediff"
const APIVersion = "0.0.1"
// PublicStateDiffAPI provides an RPC interface
// that can be used to fetch historical diffs from LevelDB directly
// that can be used to fetch historical diffs from database directly
type PublicStateDiffAPI struct {
sds *Service
}

View File

@ -20,14 +20,13 @@ import (
"fmt"
"math/big"
"github.com/cerc-io/leveldb-ethdb-rpc/pkg/client"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/triedb"
)
// Reader interface required by the statediffing service
@ -40,46 +39,36 @@ type Reader interface {
GetLatestHeader() (*types.Header, error)
}
// LvlDBReader exposes the necessary Reader methods on lvldb
type LvlDBReader struct {
// EthDBReader exposes the necessary Reader methods on an ethdb
type EthDBReader struct {
ethDB ethdb.Database
stateDB state.Database
chainConfig *params.ChainConfig
}
// LvLDBReaderConfig struct for initializing a LvlDBReader
type LvLDBReaderConfig struct {
TrieConfig *trie.Config
type EthDBReaderConfig struct {
TrieConfig *triedb.Config
ChainConfig *params.ChainConfig
Mode string
Path, AncientPath, Url string
DBCacheSize int
}
// NewLvlDBReader creates a new Reader using LevelDB
func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
var edb ethdb.Database
var err error
switch conf.Mode {
case "local":
edb, err = rawdb.NewLevelDBDatabase(conf.Path, conf.DBCacheSize, 256, "eth-statediff-service", true)
if err != nil {
return nil, err
}
edb, err = rawdb.NewDatabaseWithFreezer(edb, conf.AncientPath, "eth-statediff-service", true)
if err != nil {
return nil, err
}
case "remote":
edb, err = client.NewDatabaseClient(conf.Url)
if err != nil {
return nil, err
}
// NewEthDBReader creates a new Reader using LevelDB
func NewEthDBReader(conf EthDBReaderConfig) (*EthDBReader, error) {
opts := rawdb.OpenOptions{
Directory: conf.Path,
AncientsDirectory: conf.AncientPath,
Namespace: "eth-statediff-service",
Cache: conf.DBCacheSize,
Handles: 256,
ReadOnly: true,
}
edb, err := rawdb.Open(opts)
if err != nil {
return nil, fmt.Errorf("failed to open DB: %w", err)
}
return &LvlDBReader{
return &EthDBReader{
ethDB: edb,
stateDB: state.NewDatabaseWithConfig(edb, conf.TrieConfig),
chainConfig: conf.ChainConfig,
@ -87,7 +76,7 @@ func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
}
// GetBlockByHash gets block by hash
func (ldr *LvlDBReader) GetBlockByHash(hash common.Hash) (*types.Block, error) {
func (ldr *EthDBReader) GetBlockByHash(hash common.Hash) (*types.Block, error) {
height := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
if height == nil {
return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
@ -99,7 +88,7 @@ func (ldr *LvlDBReader) GetBlockByHash(hash common.Hash) (*types.Block, error) {
return block, nil
}
func (ldr *LvlDBReader) GetBlockByNumber(number uint64) (*types.Block, error) {
func (ldr *EthDBReader) GetBlockByNumber(number uint64) (*types.Block, error) {
hash := rawdb.ReadCanonicalHash(ldr.ethDB, number)
block := rawdb.ReadBlock(ldr.ethDB, hash, number)
if block == nil {
@ -109,12 +98,16 @@ func (ldr *LvlDBReader) GetBlockByNumber(number uint64) (*types.Block, error) {
}
// GetReceiptsByHash gets receipt by hash
func (ldr *LvlDBReader) GetReceiptsByHash(hash common.Hash) (types.Receipts, error) {
func (ldr *EthDBReader) GetReceiptsByHash(hash common.Hash) (types.Receipts, error) {
number := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
if number == nil {
return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
}
receipts := rawdb.ReadReceipts(ldr.ethDB, hash, *number, ldr.chainConfig)
header := rawdb.ReadHeader(ldr.ethDB, hash, *number)
if header == nil {
return nil, fmt.Errorf("unable to read header for header hash %s", hash)
}
receipts := rawdb.ReadReceipts(ldr.ethDB, hash, *number, header.Time, ldr.chainConfig)
if receipts == nil {
return nil, fmt.Errorf("unable to read receipts at height %d hash %s", number, hash)
}
@ -122,7 +115,7 @@ func (ldr *LvlDBReader) GetReceiptsByHash(hash common.Hash) (types.Receipts, err
}
// GetTdByHash gets td by hash
func (ldr *LvlDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
func (ldr *EthDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
number := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
if number == nil {
return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
@ -135,12 +128,12 @@ func (ldr *LvlDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
}
// StateDB returns the underlying statedb
func (ldr *LvlDBReader) StateDB() state.Database {
func (ldr *EthDBReader) StateDB() state.Database {
return ldr.stateDB
}
// GetLatestHeader gets the latest header from the levelDB
func (ldr *LvlDBReader) GetLatestHeader() (*types.Header, error) {
func (ldr *EthDBReader) GetLatestHeader() (*types.Header, error) {
header := rawdb.ReadHeadHeader(ldr.ethDB)
if header == nil {
return nil, errors.New("unable to read head header")

View File

@ -43,7 +43,7 @@ type Service struct {
// Used to build the state diff objects
builder statediff.Builder
// Used to read data from LevelDB
lvlDBReader Reader
reader Reader
// Used to signal shutdown of the service
quitChan chan struct{}
// Interface for publishing statediffs as PG-IPLD objects
@ -57,19 +57,19 @@ type Service struct {
}
// NewStateDiffService creates a new Service
func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer, conf ServiceConfig) *Service {
builder := statediff.NewBuilder(adapt.GethStateView(lvlDBReader.StateDB()))
func NewStateDiffService(reader Reader, indexer interfaces.StateDiffIndexer, conf ServiceConfig) *Service {
builder := statediff.NewBuilder(adapt.GethStateView(reader.StateDB()))
builder.SetSubtrieWorkers(conf.TrieWorkers)
if conf.WorkerQueueSize == 0 {
conf.WorkerQueueSize = defaultQueueSize
}
return &Service{
lvlDBReader: lvlDBReader,
builder: builder,
indexer: indexer,
workers: conf.ServiceWorkers,
queue: make(chan RangeRequest, conf.WorkerQueueSize),
preruns: conf.PreRuns,
reader: reader,
builder: builder,
indexer: indexer,
workers: conf.ServiceWorkers,
queue: make(chan RangeRequest, conf.WorkerQueueSize),
preruns: conf.PreRuns,
}
}
@ -224,7 +224,7 @@ func (sds *Service) Loop(wg *sync.WaitGroup) error {
// StateDiffAt returns a state diff object payload at the specific blockheight
// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
currentBlock, err := sds.reader.GetBlockByNumber(blockNumber)
if err != nil {
return nil, err
}
@ -236,7 +236,7 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*s
if blockNumber == 0 {
return sds.processStateDiff(currentBlock, common.Hash{}, params)
}
parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
if err != nil {
return nil, err
}
@ -246,7 +246,7 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*s
// StateDiffFor returns a state diff object payload for the specific blockhash
// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params) (*statediff.Payload, error) {
currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
currentBlock, err := sds.reader.GetBlockByHash(blockHash)
if err != nil {
return nil, err
}
@ -258,7 +258,7 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params)
if currentBlock.NumberU64() == 0 {
return sds.processStateDiff(currentBlock, common.Hash{}, params)
}
parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
if err != nil {
return nil, err
}
@ -297,14 +297,14 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params st
}
if params.IncludeTD {
var err error
payload.TotalDifficulty, err = sds.lvlDBReader.GetTdByHash(block.Hash())
payload.TotalDifficulty, err = sds.reader.GetTdByHash(block.Hash())
if err != nil {
return nil, err
}
}
if params.IncludeReceipts {
receiptBuff := new(bytes.Buffer)
receipts, err := sds.lvlDBReader.GetReceiptsByHash(block.Hash())
receipts, err := sds.reader.GetReceiptsByHash(block.Hash())
if err != nil {
return nil, err
}
@ -335,7 +335,7 @@ func (sds *Service) Stop() error {
func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error {
logrus.Infof("Writing state diff at block %d", blockNumber)
t := time.Now()
currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
currentBlock, err := sds.reader.GetBlockByNumber(blockNumber)
if err != nil {
return err
}
@ -345,7 +345,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params
parentRoot := common.Hash{}
if blockNumber != 0 {
parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
if err != nil {
return err
}
@ -360,7 +360,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params
func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params statediff.Params) error {
logrus.Infof("Writing state diff for block %s", blockHash)
t := time.Now()
currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
currentBlock, err := sds.reader.GetBlockByHash(blockHash)
if err != nil {
return err
}
@ -370,7 +370,7 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params statediff.Pa
parentRoot := common.Hash{}
if currentBlock.NumberU64() != 0 {
parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
if err != nil {
return err
}
@ -385,13 +385,13 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
var receipts types.Receipts
var err error
if params.IncludeTD {
totalDifficulty, err = sds.lvlDBReader.GetTdByHash(block.Hash())
totalDifficulty, err = sds.reader.GetTdByHash(block.Hash())
}
if err != nil {
return err
}
if params.IncludeReceipts {
receipts, err = sds.lvlDBReader.GetReceiptsByHash(block.Hash())
receipts, err = sds.reader.GetReceiptsByHash(block.Hash())
}
if err != nil {
return err

View File

@ -6,8 +6,8 @@
# Configure the input data using environment vars.
(
set -u
: $LEVELDB_PATH
: $LEVELDB_ANCIENT
: $ETHDB_PATH
: $ETHDB_ANCIENT
: $ETH_GENESIS_BLOCK
: $ETH_CHAIN_CONFIG
) || exit 1

View File

@ -1,21 +0,0 @@
{
"chainId": 1212,
"homesteadBlock": 0,
"eip150Block": 0,
"eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"eip155Block": 0,
"eip158Block": 0,
"byzantiumBlock": 0,
"constantinopleBlock": 0,
"petersburgBlock": 0,
"istanbulBlock": 0,
"berlinBlock": 0,
"londonBlock": 0,
"mergeForkBlock": 0,
"terminalTotalDifficulty": 0,
"mergeNetsplitBlock": 0,
"clique": {
"period": 5,
"epoch": 30000
}
}

View File

@ -1,8 +1,4 @@
# Config file for service in compose.yml
[leveldb]
mode = "local"
url = "http://127.0.0.1:8082/"
# Config file for github workflow
[server]
ipcPath = ".ipc"
@ -30,7 +26,6 @@
trie = 1024
[ethereum]
chainConfig = "test/ci-chain.json"
nodeID = "test-node"
clientName = "test-eth-statediff-service"
networkID = "test-network"

View File

@ -3,7 +3,7 @@ services:
restart: on-failure
depends_on:
- ipld-eth-db
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.3.0-alpha
environment:
DATABASE_USER: "vdbm"
DATABASE_NAME: "cerc_testing"