support pebble ethdb (geth 1.12 default), rm leveldb-ethdb-rpc
This commit is contained in:
parent
04990c0aaa
commit
7db6368ed5
@ -35,8 +35,8 @@ jobs:
|
|||||||
- name: Run basic integration test
|
- name: Run basic integration test
|
||||||
env:
|
env:
|
||||||
DATABASE_TYPE: postgres
|
DATABASE_TYPE: postgres
|
||||||
LEVELDB_PATH: ./fixtures/chaindata/_data/small2
|
ETHDB_PATH: ./fixtures/chaindata/_data/small2
|
||||||
LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
|
ETHDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
|
||||||
LOG_FILE: ./server-log
|
LOG_FILE: ./server-log
|
||||||
ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
|
ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
|
||||||
timeout-minutes: 20
|
timeout-minutes: 20
|
||||||
@ -97,8 +97,8 @@ jobs:
|
|||||||
- name: Compare statediff output
|
- name: Compare statediff output
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
env:
|
env:
|
||||||
LEVELDB_PATH: ./fixtures/chaindata/_data/small2
|
ETHDB_PATH: ./fixtures/chaindata/_data/small2
|
||||||
LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
|
ETHDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
|
||||||
ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
|
ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
|
||||||
ETH_CHAIN_CONFIG: ./eth-statediff-service/test/ci-chain.json
|
ETH_CHAIN_CONFIG: ./eth-statediff-service/test/ci-chain.json
|
||||||
run: |
|
run: |
|
||||||
|
10
README.md
10
README.md
@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
[![Go Report Card](https://goreportcard.com/badge/github.com/cerc-io/eth-statediff-service)](https://goreportcard.com/report/github.com/cerc-io/eth-statediff-service)
|
[![Go Report Card](https://goreportcard.com/badge/github.com/cerc-io/eth-statediff-service)](https://goreportcard.com/report/github.com/cerc-io/eth-statediff-service)
|
||||||
|
|
||||||
A standalone statediffing service which runs directly on top of a `go-ethereum` LevelDB instance.
|
A standalone statediffing service which runs directly on top of a `go-ethereum` database instance.
|
||||||
This service can serve historical state data over the same rpc interface as
|
This service can serve historical state data over the same rpc interface as
|
||||||
[statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.
|
[statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.
|
||||||
|
|
||||||
@ -19,7 +19,7 @@ go build .
|
|||||||
See [./environments/example.toml](./environments/example.toml) for an annotated example config file.
|
See [./environments/example.toml](./environments/example.toml) for an annotated example config file.
|
||||||
|
|
||||||
> **Note:** previous versions of this service used different variable names. To update, change the following:
|
> **Note:** previous versions of this service used different variable names. To update, change the following:
|
||||||
> * `LVLDB_MODE`, `LVLDB_PATH`, `LVLDB_ANCIENT`, `LVLDB_URL` => `LEVELDB_*`
|
> * `LVLDB_*`, `LEVELDB_*` => `ETHDB_*`
|
||||||
> * `LOG_FILE_PATH` => `LOG_FILE`
|
> * `LOG_FILE_PATH` => `LOG_FILE`
|
||||||
|
|
||||||
### Local Setup
|
### Local Setup
|
||||||
@ -208,9 +208,9 @@ ranges and params in the `prerun` section of the config.
|
|||||||
|
|
||||||
### Stats
|
### Stats
|
||||||
|
|
||||||
The binary includes a `stats` command which reports stats for the offline or remote levelDB.
|
The binary includes a `stats` command which reports stats for the offline DB.
|
||||||
|
|
||||||
At this time, the only stat supported is to return the latest/highest block height and hash found the levelDB, this is
|
At this time, the only stat supported is to return the latest/highest block height and hash found in the EthDB. This is
|
||||||
useful for determining what the upper limit is for a standalone statediffing process on a given levelDB.
|
useful for determining what the upper limit is for a standalone statediffing process using a given EthDB.
|
||||||
|
|
||||||
`./eth-statediff-service stats --config={path to toml config file}`
|
`./eth-statediff-service stats --config={path to toml config file}`
|
||||||
|
14
cmd/env.go
14
cmd/env.go
@ -30,10 +30,9 @@ const (
|
|||||||
|
|
||||||
DB_CACHE_SIZE_MB = "DB_CACHE_SIZE_MB"
|
DB_CACHE_SIZE_MB = "DB_CACHE_SIZE_MB"
|
||||||
TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
|
TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
|
||||||
LEVELDB_MODE = "LEVELDB_MODE"
|
// ETHDB_ENGINE = "ETHDB_ENGINE"
|
||||||
LEVELDB_PATH = "LEVELDB_PATH"
|
ETHDB_PATH = "ETHDB_PATH"
|
||||||
LEVELDB_ANCIENT = "LEVELDB_ANCIENT"
|
ETHDB_ANCIENT = "ETHDB_ANCIENT"
|
||||||
LEVELDB_URL = "LEVELDB_URL"
|
|
||||||
|
|
||||||
STATEDIFF_PRERUN = "STATEDIFF_PRERUN"
|
STATEDIFF_PRERUN = "STATEDIFF_PRERUN"
|
||||||
STATEDIFF_TRIE_WORKERS = "STATEDIFF_TRIE_WORKERS"
|
STATEDIFF_TRIE_WORKERS = "STATEDIFF_TRIE_WORKERS"
|
||||||
@ -119,10 +118,9 @@ func init() {
|
|||||||
viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
|
viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
|
||||||
viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)
|
viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)
|
||||||
|
|
||||||
viper.BindEnv("leveldb.mode", LEVELDB_MODE)
|
// viper.BindEnv("ethdb.engine", ETHDB_ENGINE)
|
||||||
viper.BindEnv("leveldb.path", LEVELDB_PATH)
|
viper.BindEnv("ethdb.path", ETHDB_PATH)
|
||||||
viper.BindEnv("leveldb.ancient", LEVELDB_ANCIENT)
|
viper.BindEnv("ethdb.ancient", ETHDB_ANCIENT)
|
||||||
viper.BindEnv("leveldb.url", LEVELDB_URL)
|
|
||||||
|
|
||||||
viper.BindEnv("prom.metrics", PROM_METRICS)
|
viper.BindEnv("prom.metrics", PROM_METRICS)
|
||||||
viper.BindEnv("prom.http", PROM_HTTP)
|
viper.BindEnv("prom.http", PROM_HTTP)
|
||||||
|
13
cmd/root.go
13
cmd/root.go
@ -115,10 +115,9 @@ func init() {
|
|||||||
rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(),
|
rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(),
|
||||||
"log level (trace, debug, info, warn, error, fatal, panic")
|
"log level (trace, debug, info, warn, error, fatal, panic")
|
||||||
|
|
||||||
rootCmd.PersistentFlags().String("leveldb-mode", "local", "LevelDB access mode (local, remote)")
|
// rootCmd.PersistentFlags().String("ethdb-engine", "local", "Ethdb engine type (leveldb, pebble)")
|
||||||
rootCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
|
rootCmd.PersistentFlags().String("ethdb-path", "", "path to primary datastore")
|
||||||
rootCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
|
rootCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
|
||||||
rootCmd.PersistentFlags().String("leveldb-url", "", "url to primary leveldb-ethdb-rpc server")
|
|
||||||
|
|
||||||
rootCmd.PersistentFlags().Bool("prerun", false, "turn on prerun of toml configured ranges")
|
rootCmd.PersistentFlags().Bool("prerun", false, "turn on prerun of toml configured ranges")
|
||||||
rootCmd.PersistentFlags().Int("service-workers", 1, "number of range requests to process concurrently")
|
rootCmd.PersistentFlags().Int("service-workers", 1, "number of range requests to process concurrently")
|
||||||
@ -178,10 +177,10 @@ func init() {
|
|||||||
viper.BindPFlag("statediff.trieWorkers", rootCmd.PersistentFlags().Lookup("trie-workers"))
|
viper.BindPFlag("statediff.trieWorkers", rootCmd.PersistentFlags().Lookup("trie-workers"))
|
||||||
viper.BindPFlag("statediff.workerQueueSize", rootCmd.PersistentFlags().Lookup("worker-queue-size"))
|
viper.BindPFlag("statediff.workerQueueSize", rootCmd.PersistentFlags().Lookup("worker-queue-size"))
|
||||||
|
|
||||||
viper.BindPFlag("leveldb.mode", rootCmd.PersistentFlags().Lookup("leveldb-mode"))
|
// viper.BindPFlag("ethdb.engine", rootCmd.PersistentFlags().Lookup("ethdb-engine"))
|
||||||
viper.BindPFlag("leveldb.path", rootCmd.PersistentFlags().Lookup("leveldb-path"))
|
viper.BindPFlag("ethdb.path", rootCmd.PersistentFlags().Lookup("ethdb-path"))
|
||||||
viper.BindPFlag("leveldb.ancient", rootCmd.PersistentFlags().Lookup("ancient-path"))
|
viper.BindPFlag("ethdb.ancient", rootCmd.PersistentFlags().Lookup("ancient-path"))
|
||||||
viper.BindPFlag("leveldb.url", rootCmd.PersistentFlags().Lookup("leveldb-url"))
|
viper.BindPFlag("ethdb.url", rootCmd.PersistentFlags().Lookup("ethdb-url"))
|
||||||
|
|
||||||
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
|
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
|
||||||
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
|
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
|
||||||
|
@ -35,8 +35,8 @@ import (
|
|||||||
// serveCmd represents the serve command
|
// serveCmd represents the serve command
|
||||||
var serveCmd = &cobra.Command{
|
var serveCmd = &cobra.Command{
|
||||||
Use: "serve",
|
Use: "serve",
|
||||||
Short: "Stand up a standalone statediffing RPC service on top of LevelDB",
|
Short: "Standalone statediffing RPC service on top of an Ethereum database",
|
||||||
Long: `Usage
|
Long: `Usage:
|
||||||
|
|
||||||
./eth-statediff-service serve --config={path to toml config file}`,
|
./eth-statediff-service serve --config={path to toml config file}`,
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
@ -63,7 +63,7 @@ func serve() {
|
|||||||
logWithCommand.Debug("Running eth-statediff-service serve command")
|
logWithCommand.Debug("Running eth-statediff-service serve command")
|
||||||
logWithCommand.Debugf("Parallelism: %d", maxParallelism())
|
logWithCommand.Debugf("Parallelism: %d", maxParallelism())
|
||||||
|
|
||||||
reader, chainConf, nodeInfo := instantiateLevelDBReader()
|
reader, chainConf, nodeInfo := createReader()
|
||||||
|
|
||||||
reportLatestBlock(reader)
|
reportLatestBlock(reader)
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ import (
|
|||||||
// statsCmd represents the serve command
|
// statsCmd represents the serve command
|
||||||
var statsCmd = &cobra.Command{
|
var statsCmd = &cobra.Command{
|
||||||
Use: "stats",
|
Use: "stats",
|
||||||
Short: "Report stats for cold levelDB",
|
Short: "Report stats for cold DB",
|
||||||
Long: `Usage
|
Long: `Usage
|
||||||
|
|
||||||
./eth-statediff-service stats --config={path to toml config file}`,
|
./eth-statediff-service stats --config={path to toml config file}`,
|
||||||
@ -41,6 +41,6 @@ func init() {
|
|||||||
func stats() {
|
func stats() {
|
||||||
logWithCommand.Info("Running eth-statediff-service stats command")
|
logWithCommand.Info("Running eth-statediff-service stats command")
|
||||||
|
|
||||||
reader, _, _ := instantiateLevelDBReader()
|
reader, _, _ := createReader()
|
||||||
reportLatestBlock(reader)
|
reportLatestBlock(reader)
|
||||||
}
|
}
|
||||||
|
39
cmd/util.go
39
cmd/util.go
@ -87,24 +87,18 @@ func setupPreRunRanges() []pkg.RangeRequest {
|
|||||||
return blockRanges
|
return blockRanges
|
||||||
}
|
}
|
||||||
|
|
||||||
func instantiateLevelDBReader() (pkg.Reader, *params.ChainConfig, node.Info) {
|
func createReader() (pkg.Reader, *params.ChainConfig, node.Info) {
|
||||||
// load some necessary params
|
// load some necessary params
|
||||||
logWithCommand.Debug("Loading statediff service parameters")
|
logWithCommand.Debug("Loading statediff service parameters")
|
||||||
mode := viper.GetString("leveldb.mode")
|
path := viper.GetString("ethdb.path")
|
||||||
path := viper.GetString("leveldb.path")
|
ancientPath := viper.GetString("ethdb.ancient")
|
||||||
ancientPath := viper.GetString("leveldb.ancient")
|
url := viper.GetString("ethdb.url")
|
||||||
url := viper.GetString("leveldb.url")
|
|
||||||
|
|
||||||
if mode == "local" {
|
if path == "" {
|
||||||
if path == "" || ancientPath == "" {
|
logWithCommand.Fatal("Require a valid Ethereum chain data path")
|
||||||
logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
|
}
|
||||||
}
|
if ancientPath == "" {
|
||||||
} else if mode == "remote" {
|
ancientPath = path + "/ancient"
|
||||||
if url == "" {
|
|
||||||
logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
logWithCommand.Fatal("Invalid mode provided for LevelDB access")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
nodeInfo := getEthNodeInfo()
|
nodeInfo := getEthNodeInfo()
|
||||||
@ -115,9 +109,8 @@ func instantiateLevelDBReader() (pkg.Reader, *params.ChainConfig, node.Info) {
|
|||||||
logWithCommand.Fatalf("Unable to instantiate chain config: %s", err)
|
logWithCommand.Fatalf("Unable to instantiate chain config: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// create LevelDB reader
|
logWithCommand.Debug("Creating DB reader")
|
||||||
logWithCommand.Debug("Creating LevelDB reader")
|
readerConf := pkg.EthDBReaderConfig{
|
||||||
readerConf := pkg.LvLDBReaderConfig{
|
|
||||||
TrieConfig: &triedb.Config{
|
TrieConfig: &triedb.Config{
|
||||||
Preimages: false,
|
Preimages: false,
|
||||||
IsVerkle: false,
|
IsVerkle: false,
|
||||||
@ -126,15 +119,15 @@ func instantiateLevelDBReader() (pkg.Reader, *params.ChainConfig, node.Info) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
ChainConfig: chainConf,
|
ChainConfig: chainConf,
|
||||||
Mode: mode,
|
// Mode: mode,
|
||||||
Path: path,
|
Path: path,
|
||||||
AncientPath: ancientPath,
|
AncientPath: ancientPath,
|
||||||
Url: url,
|
Url: url,
|
||||||
DBCacheSize: viper.GetInt("cache.database"),
|
DBCacheSize: viper.GetInt("cache.database"),
|
||||||
}
|
}
|
||||||
reader, err := pkg.NewLvlDBReader(readerConf)
|
reader, err := pkg.NewEthDBReader(readerConf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logWithCommand.Fatalf("Unable to instantiate levelDB reader: %s", err)
|
logWithCommand.Fatalf("Unable to instantiate DB reader: %s", err)
|
||||||
}
|
}
|
||||||
return reader, chainConf, nodeInfo
|
return reader, chainConf, nodeInfo
|
||||||
}
|
}
|
||||||
@ -146,10 +139,10 @@ func reportLatestBlock(reader pkg.Reader) {
|
|||||||
logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
|
logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
|
||||||
}
|
}
|
||||||
if header.Number == nil {
|
if header.Number == nil {
|
||||||
logWithCommand.Fatal("Latest header found in levelDB has a nil block height")
|
logWithCommand.Fatal("Latest header found in DB has a nil block height")
|
||||||
}
|
}
|
||||||
logWithCommand.
|
logWithCommand.
|
||||||
WithField("height", header.Number).
|
WithField("height", header.Number).
|
||||||
WithField("hash", header.Hash()).
|
WithField("hash", header.Hash()).
|
||||||
Info("Latest block found in levelDB")
|
Info("Latest block found in DB")
|
||||||
}
|
}
|
||||||
|
@ -1,8 +1,6 @@
|
|||||||
[leveldb]
|
[ethdb]
|
||||||
mode = "local" # LEVELDB_MODE
|
path = "/app/geth-rw/chaindata" # ETHDB_PATH
|
||||||
path = "/app/geth-rw/chaindata" # LEVELDB_PATH
|
ancient = "/app/geth-rw/chaindata/ancient" # ETHDB_ANCIENT
|
||||||
ancient = "/app/geth-rw/chaindata/ancient" # LEVELDB_ANCIENT
|
|
||||||
url = "http://127.0.0.1:8082/" # LEVELDB_URL
|
|
||||||
|
|
||||||
[server]
|
[server]
|
||||||
ipcPath = ".ipc" # SERVICE_IPC_PATH
|
ipcPath = ".ipc" # SERVICE_IPC_PATH
|
||||||
|
@ -1,13 +1,10 @@
|
|||||||
[leveldb]
|
[ethdb]
|
||||||
# LevelDB access mode <local | remote>
|
# # access mode <local | remote>
|
||||||
mode = "local" # LEVELDB_MODE
|
# mode = "local" # ETHDB_MODE
|
||||||
|
|
||||||
# LevelDB paths (local mode)
|
# Ethdb paths (local mode)
|
||||||
path = "/Users/user/Library/Ethereum/geth/chaindata" # LEVELDB_PATH
|
path = "/Users/user/Library/Ethereum/geth/chaindata" # ETHDB_PATH
|
||||||
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT
|
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # ETHDB_ANCIENT
|
||||||
|
|
||||||
# URL for leveldb-ethdb-rpc endpoint (remote mode)
|
|
||||||
url = "http://127.0.0.1:8082/" # LEVELDB_URL
|
|
||||||
|
|
||||||
[server]
|
[server]
|
||||||
ipcPath = ".ipc" # SERVICE_IPC_PATH
|
ipcPath = ".ipc" # SERVICE_IPC_PATH
|
||||||
|
3
go.mod
3
go.mod
@ -3,7 +3,6 @@ module github.com/cerc-io/eth-statediff-service
|
|||||||
go 1.21
|
go 1.21
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/cerc-io/leveldb-ethdb-rpc v1.1.13
|
|
||||||
github.com/cerc-io/plugeth-statediff v0.3.0
|
github.com/cerc-io/plugeth-statediff v0.3.0
|
||||||
github.com/ethereum/go-ethereum v1.14.5
|
github.com/ethereum/go-ethereum v1.14.5
|
||||||
github.com/prometheus/client_golang v1.16.0
|
github.com/prometheus/client_golang v1.16.0
|
||||||
@ -147,5 +146,3 @@ require (
|
|||||||
lukechampine.com/blake3 v1.2.1 // indirect
|
lukechampine.com/blake3 v1.2.1 // indirect
|
||||||
rsc.io/tmplfunc v0.0.3 // indirect
|
rsc.io/tmplfunc v0.0.3 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
replace github.com/cerc-io/leveldb-ethdb-rpc => git.vdb.to/cerc-io/leveldb-ethdb-rpc v1.1.14-0.20240424110216-4a9a0d18ee8a
|
|
||||||
|
2
go.sum
2
go.sum
@ -1,5 +1,3 @@
|
|||||||
git.vdb.to/cerc-io/leveldb-ethdb-rpc v1.1.14-0.20240424110216-4a9a0d18ee8a h1:YLqeScSatqrInROUubCCY9cQ1XBYAFx/Ulo6UgJmbhk=
|
|
||||||
git.vdb.to/cerc-io/leveldb-ethdb-rpc v1.1.14-0.20240424110216-4a9a0d18ee8a/go.mod h1:XWPmT4ZtMlAaz9bL58Qfaf+msHTawtCIQUhT/feN6Kg=
|
|
||||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
|
github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
|
||||||
github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||||
|
@ -28,7 +28,7 @@ const APIName = "statediff"
|
|||||||
const APIVersion = "0.0.1"
|
const APIVersion = "0.0.1"
|
||||||
|
|
||||||
// PublicStateDiffAPI provides an RPC interface
|
// PublicStateDiffAPI provides an RPC interface
|
||||||
// that can be used to fetch historical diffs from LevelDB directly
|
// that can be used to fetch historical diffs from database directly
|
||||||
type PublicStateDiffAPI struct {
|
type PublicStateDiffAPI struct {
|
||||||
sds *Service
|
sds *Service
|
||||||
}
|
}
|
||||||
|
@ -20,7 +20,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"math/big"
|
"math/big"
|
||||||
|
|
||||||
"github.com/cerc-io/leveldb-ethdb-rpc/pkg/client"
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||||
"github.com/ethereum/go-ethereum/core/state"
|
"github.com/ethereum/go-ethereum/core/state"
|
||||||
@ -40,46 +39,38 @@ type Reader interface {
|
|||||||
GetLatestHeader() (*types.Header, error)
|
GetLatestHeader() (*types.Header, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// LvlDBReader exposes the necessary Reader methods on lvldb
|
// EthDBReader exposes the necessary Reader methods on an ethdb
|
||||||
type LvlDBReader struct {
|
type EthDBReader struct {
|
||||||
ethDB ethdb.Database
|
ethDB ethdb.Database
|
||||||
stateDB state.Database
|
stateDB state.Database
|
||||||
chainConfig *params.ChainConfig
|
chainConfig *params.ChainConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
// LvLDBReaderConfig struct for initializing a LvlDBReader
|
type EthDBReaderConfig struct {
|
||||||
type LvLDBReaderConfig struct {
|
TrieConfig *triedb.Config
|
||||||
TrieConfig *triedb.Config
|
ChainConfig *params.ChainConfig
|
||||||
ChainConfig *params.ChainConfig
|
// Mode string
|
||||||
Mode string
|
|
||||||
Path, AncientPath, Url string
|
Path, AncientPath, Url string
|
||||||
DBCacheSize int
|
DBCacheSize int
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewLvlDBReader creates a new Reader using LevelDB
|
// NewEthDBReader creates a new Reader using LevelDB
|
||||||
func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
|
func NewEthDBReader(conf EthDBReaderConfig) (*EthDBReader, error) {
|
||||||
var edb ethdb.Database
|
opts := rawdb.OpenOptions{
|
||||||
var err error
|
// Type: conf.Mode,
|
||||||
|
Directory: conf.Path,
|
||||||
switch conf.Mode {
|
AncientsDirectory: conf.AncientPath,
|
||||||
case "local":
|
Namespace: "eth-statediff-service",
|
||||||
edb, err = rawdb.NewLevelDBDatabase(conf.Path, conf.DBCacheSize, 256, "eth-statediff-service", true)
|
Cache: conf.DBCacheSize,
|
||||||
if err != nil {
|
Handles: 256,
|
||||||
return nil, err
|
ReadOnly: true,
|
||||||
}
|
}
|
||||||
|
edb, err := rawdb.Open(opts)
|
||||||
edb, err = rawdb.NewDatabaseWithFreezer(edb, conf.AncientPath, "eth-statediff-service", true)
|
if err != nil {
|
||||||
if err != nil {
|
return nil, fmt.Errorf("failed to open DB: %w", err)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
case "remote":
|
|
||||||
edb, err = client.NewDatabaseClient(conf.Url)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return &LvlDBReader{
|
return &EthDBReader{
|
||||||
ethDB: edb,
|
ethDB: edb,
|
||||||
stateDB: state.NewDatabaseWithConfig(edb, conf.TrieConfig),
|
stateDB: state.NewDatabaseWithConfig(edb, conf.TrieConfig),
|
||||||
chainConfig: conf.ChainConfig,
|
chainConfig: conf.ChainConfig,
|
||||||
@ -87,7 +78,7 @@ func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetBlockByHash gets block by hash
|
// GetBlockByHash gets block by hash
|
||||||
func (ldr *LvlDBReader) GetBlockByHash(hash common.Hash) (*types.Block, error) {
|
func (ldr *EthDBReader) GetBlockByHash(hash common.Hash) (*types.Block, error) {
|
||||||
height := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
|
height := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
|
||||||
if height == nil {
|
if height == nil {
|
||||||
return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
|
return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
|
||||||
@ -99,7 +90,7 @@ func (ldr *LvlDBReader) GetBlockByHash(hash common.Hash) (*types.Block, error) {
|
|||||||
return block, nil
|
return block, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ldr *LvlDBReader) GetBlockByNumber(number uint64) (*types.Block, error) {
|
func (ldr *EthDBReader) GetBlockByNumber(number uint64) (*types.Block, error) {
|
||||||
hash := rawdb.ReadCanonicalHash(ldr.ethDB, number)
|
hash := rawdb.ReadCanonicalHash(ldr.ethDB, number)
|
||||||
block := rawdb.ReadBlock(ldr.ethDB, hash, number)
|
block := rawdb.ReadBlock(ldr.ethDB, hash, number)
|
||||||
if block == nil {
|
if block == nil {
|
||||||
@ -109,7 +100,7 @@ func (ldr *LvlDBReader) GetBlockByNumber(number uint64) (*types.Block, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetReceiptsByHash gets receipt by hash
|
// GetReceiptsByHash gets receipt by hash
|
||||||
func (ldr *LvlDBReader) GetReceiptsByHash(hash common.Hash) (types.Receipts, error) {
|
func (ldr *EthDBReader) GetReceiptsByHash(hash common.Hash) (types.Receipts, error) {
|
||||||
number := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
|
number := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
|
||||||
if number == nil {
|
if number == nil {
|
||||||
return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
|
return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
|
||||||
@ -126,7 +117,7 @@ func (ldr *LvlDBReader) GetReceiptsByHash(hash common.Hash) (types.Receipts, err
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetTdByHash gets td by hash
|
// GetTdByHash gets td by hash
|
||||||
func (ldr *LvlDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
|
func (ldr *EthDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
|
||||||
number := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
|
number := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
|
||||||
if number == nil {
|
if number == nil {
|
||||||
return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
|
return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
|
||||||
@ -139,12 +130,12 @@ func (ldr *LvlDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// StateDB returns the underlying statedb
|
// StateDB returns the underlying statedb
|
||||||
func (ldr *LvlDBReader) StateDB() state.Database {
|
func (ldr *EthDBReader) StateDB() state.Database {
|
||||||
return ldr.stateDB
|
return ldr.stateDB
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetLatestHeader gets the latest header from the levelDB
|
// GetLatestHeader gets the latest header from the levelDB
|
||||||
func (ldr *LvlDBReader) GetLatestHeader() (*types.Header, error) {
|
func (ldr *EthDBReader) GetLatestHeader() (*types.Header, error) {
|
||||||
header := rawdb.ReadHeadHeader(ldr.ethDB)
|
header := rawdb.ReadHeadHeader(ldr.ethDB)
|
||||||
if header == nil {
|
if header == nil {
|
||||||
return nil, errors.New("unable to read head header")
|
return nil, errors.New("unable to read head header")
|
||||||
|
@ -43,7 +43,7 @@ type Service struct {
|
|||||||
// Used to build the state diff objects
|
// Used to build the state diff objects
|
||||||
builder statediff.Builder
|
builder statediff.Builder
|
||||||
// Used to read data from LevelDB
|
// Used to read data from LevelDB
|
||||||
lvlDBReader Reader
|
reader Reader
|
||||||
// Used to signal shutdown of the service
|
// Used to signal shutdown of the service
|
||||||
quitChan chan struct{}
|
quitChan chan struct{}
|
||||||
// Interface for publishing statediffs as PG-IPLD objects
|
// Interface for publishing statediffs as PG-IPLD objects
|
||||||
@ -57,19 +57,19 @@ type Service struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewStateDiffService creates a new Service
|
// NewStateDiffService creates a new Service
|
||||||
func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer, conf ServiceConfig) *Service {
|
func NewStateDiffService(reader Reader, indexer interfaces.StateDiffIndexer, conf ServiceConfig) *Service {
|
||||||
builder := statediff.NewBuilder(adapt.GethStateView(lvlDBReader.StateDB()))
|
builder := statediff.NewBuilder(adapt.GethStateView(reader.StateDB()))
|
||||||
builder.SetSubtrieWorkers(conf.TrieWorkers)
|
builder.SetSubtrieWorkers(conf.TrieWorkers)
|
||||||
if conf.WorkerQueueSize == 0 {
|
if conf.WorkerQueueSize == 0 {
|
||||||
conf.WorkerQueueSize = defaultQueueSize
|
conf.WorkerQueueSize = defaultQueueSize
|
||||||
}
|
}
|
||||||
return &Service{
|
return &Service{
|
||||||
lvlDBReader: lvlDBReader,
|
reader: reader,
|
||||||
builder: builder,
|
builder: builder,
|
||||||
indexer: indexer,
|
indexer: indexer,
|
||||||
workers: conf.ServiceWorkers,
|
workers: conf.ServiceWorkers,
|
||||||
queue: make(chan RangeRequest, conf.WorkerQueueSize),
|
queue: make(chan RangeRequest, conf.WorkerQueueSize),
|
||||||
preruns: conf.PreRuns,
|
preruns: conf.PreRuns,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -224,7 +224,7 @@ func (sds *Service) Loop(wg *sync.WaitGroup) error {
|
|||||||
// StateDiffAt returns a state diff object payload at the specific blockheight
|
// StateDiffAt returns a state diff object payload at the specific blockheight
|
||||||
// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
|
// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
|
||||||
func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
|
func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
|
||||||
currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
|
currentBlock, err := sds.reader.GetBlockByNumber(blockNumber)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -236,7 +236,7 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*s
|
|||||||
if blockNumber == 0 {
|
if blockNumber == 0 {
|
||||||
return sds.processStateDiff(currentBlock, common.Hash{}, params)
|
return sds.processStateDiff(currentBlock, common.Hash{}, params)
|
||||||
}
|
}
|
||||||
parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
|
parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -246,7 +246,7 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*s
|
|||||||
// StateDiffFor returns a state diff object payload for the specific blockhash
|
// StateDiffFor returns a state diff object payload for the specific blockhash
|
||||||
// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
|
// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
|
||||||
func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params) (*statediff.Payload, error) {
|
func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params) (*statediff.Payload, error) {
|
||||||
currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
|
currentBlock, err := sds.reader.GetBlockByHash(blockHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -258,7 +258,7 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params)
|
|||||||
if currentBlock.NumberU64() == 0 {
|
if currentBlock.NumberU64() == 0 {
|
||||||
return sds.processStateDiff(currentBlock, common.Hash{}, params)
|
return sds.processStateDiff(currentBlock, common.Hash{}, params)
|
||||||
}
|
}
|
||||||
parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
|
parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -297,14 +297,14 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params st
|
|||||||
}
|
}
|
||||||
if params.IncludeTD {
|
if params.IncludeTD {
|
||||||
var err error
|
var err error
|
||||||
payload.TotalDifficulty, err = sds.lvlDBReader.GetTdByHash(block.Hash())
|
payload.TotalDifficulty, err = sds.reader.GetTdByHash(block.Hash())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if params.IncludeReceipts {
|
if params.IncludeReceipts {
|
||||||
receiptBuff := new(bytes.Buffer)
|
receiptBuff := new(bytes.Buffer)
|
||||||
receipts, err := sds.lvlDBReader.GetReceiptsByHash(block.Hash())
|
receipts, err := sds.reader.GetReceiptsByHash(block.Hash())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -335,7 +335,7 @@ func (sds *Service) Stop() error {
|
|||||||
func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error {
|
func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error {
|
||||||
logrus.Infof("Writing state diff at block %d", blockNumber)
|
logrus.Infof("Writing state diff at block %d", blockNumber)
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
|
currentBlock, err := sds.reader.GetBlockByNumber(blockNumber)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -345,7 +345,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params
|
|||||||
|
|
||||||
parentRoot := common.Hash{}
|
parentRoot := common.Hash{}
|
||||||
if blockNumber != 0 {
|
if blockNumber != 0 {
|
||||||
parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
|
parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -360,7 +360,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params
|
|||||||
func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params statediff.Params) error {
|
func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params statediff.Params) error {
|
||||||
logrus.Infof("Writing state diff for block %s", blockHash)
|
logrus.Infof("Writing state diff for block %s", blockHash)
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
|
currentBlock, err := sds.reader.GetBlockByHash(blockHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -370,7 +370,7 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params statediff.Pa
|
|||||||
|
|
||||||
parentRoot := common.Hash{}
|
parentRoot := common.Hash{}
|
||||||
if currentBlock.NumberU64() != 0 {
|
if currentBlock.NumberU64() != 0 {
|
||||||
parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
|
parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -385,13 +385,13 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
|
|||||||
var receipts types.Receipts
|
var receipts types.Receipts
|
||||||
var err error
|
var err error
|
||||||
if params.IncludeTD {
|
if params.IncludeTD {
|
||||||
totalDifficulty, err = sds.lvlDBReader.GetTdByHash(block.Hash())
|
totalDifficulty, err = sds.reader.GetTdByHash(block.Hash())
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if params.IncludeReceipts {
|
if params.IncludeReceipts {
|
||||||
receipts, err = sds.lvlDBReader.GetReceiptsByHash(block.Hash())
|
receipts, err = sds.reader.GetReceiptsByHash(block.Hash())
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -6,8 +6,8 @@
|
|||||||
# Configure the input data using environment vars.
|
# Configure the input data using environment vars.
|
||||||
(
|
(
|
||||||
set -u
|
set -u
|
||||||
: $LEVELDB_PATH
|
: $ETHDB_PATH
|
||||||
: $LEVELDB_ANCIENT
|
: $ETHDB_ANCIENT
|
||||||
: $ETH_GENESIS_BLOCK
|
: $ETH_GENESIS_BLOCK
|
||||||
: $ETH_CHAIN_CONFIG
|
: $ETH_CHAIN_CONFIG
|
||||||
) || exit 1
|
) || exit 1
|
||||||
|
@ -1,8 +1,8 @@
|
|||||||
# Config file for service in compose.yml
|
# Config file for service in compose.yml
|
||||||
|
|
||||||
[leveldb]
|
# [ethdb]
|
||||||
mode = "local"
|
# mode = "local"
|
||||||
url = "http://127.0.0.1:8082/"
|
# url = "http://127.0.0.1:8082/"
|
||||||
|
|
||||||
[server]
|
[server]
|
||||||
ipcPath = ".ipc"
|
ipcPath = ".ipc"
|
||||||
|
Loading…
Reference in New Issue
Block a user