cerc-io/ipld-eth-state-snapshot
16
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

support pebble db

Browse Source
This commit is contained in:
Roy Crihfield 2024-06-19 13:49:54 +08:00
parent 4d9a991553
commit 1d1bfe4aa4
9 changed files with 55 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.gitea/workflows/test.yml
View File

@ -43,8 +43,8 @@ jobs:
env:
SNAPSHOT_MODE: postgres
SNAPSHOT_BLOCK_HEIGHT: 32
LEVELDB_PATH: ./fixtures/chaindata/_data/small
LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient
ETHDB_PATH: ./fixtures/chaindata/_data/small
ETHDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient
ETH_GENESIS_BLOCK: "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
run: |
until
@ -102,8 +102,8 @@ jobs:
- name: Compare snapshot output
env:
SNAPSHOT_BLOCK_HEIGHT: 200
LEVELDB_PATH: ./fixtures/chaindata/_data/small2
LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
ETHDB_PATH: ./fixtures/chaindata/_data/small2
ETHDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
run: |
until

18
README.md
View File

@ -1,6 +1,6 @@
# ipld-eth-state-snapshot
> Tool for extracting the entire Ethereum state at a particular block height from leveldb into Postgres-backed IPFS
> Tool for extracting the entire Ethereum state at a particular block height from a cold database into Postgres-backed IPFS
[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipld-eth-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/ipld-eth-state-snapshot)
@ -20,15 +20,15 @@ Config format:
[snapshot]
mode = "file" # indicates output mode <postgres | file>
workers = 4 # degree of concurrency: the state trie is subdivided into sections that are traversed and processed concurrently
blockHeight = -1 # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in leveldb)
blockHeight = -1 # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in ethdb)
recoveryFile = "recovery_file" # specifies a file to output recovery information on error or premature closure
accounts = [] # list of accounts (addresses) to take the snapshot for # SNAPSHOT_ACCOUNTS
[leveldb]
# path to geth leveldb
path = "/Users/user/Library/Ethereum/geth/chaindata" # LEVELDB_PATH
[ethdb]
# path to geth ethdb
path = "/Users/user/Library/Ethereum/geth/chaindata" # ETHDB_PATH
# path to geth ancient database
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # ETHDB_ANCIENT
[database]
# when operating in 'postgres' output mode
@ -66,14 +66,14 @@ Config format:
```
> **Note:** previous versions of this service used different variable names. To update, change the following:
> * `LVL_DB_PATH` => `LEVELDB_PATH`
> * `ANCIENT_DB_PATH` => `LEVELDB_ANCIENT`
> * `LVL_DB_PATH`, `LEVELDB_PATH` => `ETHDB_PATH`
> * `ANCIENT_DB_PATH`, `LEVELDB_ANCIENT` => `ETHDB_ANCIENT`
> * `LOGRUS_LEVEL`, `LOGRUS_FILE` => `LOG_LEVEL`, `LOG_FILE`, etc.
## Usage
* For state snapshot from LevelDB:
* For state snapshot from EthDB:
```bash
./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}

16
cmd/stateSnapshot.go
View File

@ -30,7 +30,7 @@ import (
// stateSnapshotCmd represents the stateSnapshot command
var stateSnapshotCmd = &cobra.Command{
Use: "stateSnapshot",
Short: "Extract the entire Ethereum state from leveldb and publish into PG-IPFS",
Short: "Extract the entire Ethereum state from Ethdb and publish into PG-IPFS",
Long: `Usage
./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}`,
@ -47,9 +47,9 @@ func stateSnapshot() {
if err != nil {
logWithCommand.Fatalf("unable to initialize config: %v", err)
}
logWithCommand.Infof("opening levelDB and ancient data at %s and %s",
config.Eth.LevelDBPath, config.Eth.AncientDBPath)
edb, err := snapshot.NewLevelDB(config.Eth)
logWithCommand.Infof("opening ethdb and ancient data at %s and %s",
config.Eth.DBPath, config.Eth.AncientDBPath)
edb, err := snapshot.NewEthDB(config.Eth)
if err != nil {
logWithCommand.Fatal(err)
}
@ -99,8 +99,8 @@ func stateSnapshot() {
func init() {
rootCmd.AddCommand(stateSnapshotCmd)
stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_PATH_CLI, "", "path to primary datastore")
stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_ANCIENT_CLI, "", "path to ancient datastore")
stateSnapshotCmd.PersistentFlags().String(snapshot.ETHDB_PATH_CLI, "", "path to primary datastore")
stateSnapshotCmd.PersistentFlags().String(snapshot.ETHDB_ANCIENT_CLI, "", "path to ancient datastore")
stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI, "", "block height to extract state at")
stateSnapshotCmd.PersistentFlags().Int(snapshot.SNAPSHOT_WORKERS_CLI, 1, "number of concurrent workers to use")
stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_RECOVERY_FILE_CLI, "", "file to recover from a previous iteration")
@ -108,8 +108,8 @@ func init() {
stateSnapshotCmd.PersistentFlags().String(snapshot.FILE_OUTPUT_DIR_CLI, "", "directory for writing ouput to while operating in 'file' mode")
stateSnapshotCmd.PersistentFlags().StringArray(snapshot.SNAPSHOT_ACCOUNTS_CLI, nil, "list of account addresses to limit snapshot to")
viper.BindPFlag(snapshot.LEVELDB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_PATH_CLI))
viper.BindPFlag(snapshot.LEVELDB_ANCIENT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_ANCIENT_CLI))
viper.BindPFlag(snapshot.ETHDB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ETHDB_PATH_CLI))
viper.BindPFlag(snapshot.ETHDB_ANCIENT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ETHDB_ANCIENT_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_WORKERS_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_RECOVERY_FILE_CLI))

2
environments/example.toml
View File

@ -5,7 +5,7 @@
user = "vdbm"
password = "password"
[leveldb]
[ethdb]
path = "/Users/user/go/src/github.com/cerc-io/ipld-eth-state-snapshot/fixture/chain2data"
ancient = "/Users/user/go/src/github.com/cerc-io/ipld-eth-state-snapshot/fixture/chain2data/ancient"

20
pkg/snapshot/config.go
View File

@ -41,15 +41,15 @@ const (
// Config contains params for both databases the service uses
type Config struct {
Eth *EthConfig
Eth *EthDBConfig
DB *DBConfig
File *FileConfig
Service *ServiceConfig
}
// EthConfig is config parameters for the chain.
type EthConfig struct {
LevelDBPath string
// EthDBConfig is config parameters for the chain DB.
type EthDBConfig struct {
DBPath string
AncientDBPath string
NodeInfo ethNode.Info
}
@ -67,7 +67,7 @@ type ServiceConfig struct {
func NewConfig(mode SnapshotMode) (*Config, error) {
ret := &Config{
&EthConfig{},
&EthDBConfig{},
&DBConfig{},
&FileConfig{},
&ServiceConfig{},
@ -77,7 +77,7 @@ func NewConfig(mode SnapshotMode) (*Config, error) {
func NewInPlaceSnapshotConfig() *Config {
ret := &Config{
&EthConfig{},
&EthDBConfig{},
&DBConfig{},
&FileConfig{},
&ServiceConfig{},
@ -104,11 +104,11 @@ func (c *Config) Init(mode SnapshotMode) error {
ChainID: viper.GetUint64(ETH_CHAIN_ID_TOML),
}
viper.BindEnv(LEVELDB_ANCIENT_TOML, LEVELDB_ANCIENT)
viper.BindEnv(LEVELDB_PATH_TOML, LEVELDB_PATH)
viper.BindEnv(ETHDB_ANCIENT_TOML, ETHDB_ANCIENT)
viper.BindEnv(ETHDB_PATH_TOML, ETHDB_PATH)
c.Eth.AncientDBPath = viper.GetString(LEVELDB_ANCIENT_TOML)
c.Eth.LevelDBPath = viper.GetString(LEVELDB_PATH_TOML)
c.Eth.AncientDBPath = viper.GetString(ETHDB_ANCIENT_TOML)
c.Eth.DBPath = viper.GetString(ETHDB_PATH_TOML)
switch mode {
case FileSnapshot:

12
pkg/snapshot/env.go
View File

@ -34,8 +34,8 @@ const (
FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"
LEVELDB_ANCIENT = "LEVELDB_ANCIENT"
LEVELDB_PATH = "LEVELDB_PATH"
ETHDB_ANCIENT = "ETHDB_ANCIENT"
ETHDB_PATH = "ETHDB_PATH"
ETH_CLIENT_NAME = "ETH_CLIENT_NAME"
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
@ -72,8 +72,8 @@ const (
FILE_OUTPUT_DIR_TOML = "file.outputDir"
LEVELDB_ANCIENT_TOML = "leveldb.ancient"
LEVELDB_PATH_TOML = "leveldb.path"
ETHDB_ANCIENT_TOML = "ethdb.ancient"
ETHDB_PATH_TOML = "ethdb.path"
ETH_CLIENT_NAME_TOML = "ethereum.clientName"
ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
@ -110,8 +110,8 @@ const (
FILE_OUTPUT_DIR_CLI = "output-dir"
LEVELDB_ANCIENT_CLI = "ancient-path"
LEVELDB_PATH_CLI = "leveldb-path"
ETHDB_ANCIENT_CLI = "ancient-path"
ETHDB_PATH_CLI = "ethdb-path"
ETH_CLIENT_NAME_CLI = "ethereum-client-name"
ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block"

19
pkg/snapshot/service.go
View File

@ -56,16 +56,15 @@ type Service struct {
recoveryFile string
}
func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
kvdb, err := rawdb.NewLevelDBDatabase(con.LevelDBPath, 1024, 256, "ipld-eth-state-snapshot", true)
if err != nil {
return nil, fmt.Errorf("failed to connect LevelDB: %s", err)
}
edb, err := rawdb.NewDatabaseWithFreezer(kvdb, con.AncientDBPath, "ipld-eth-state-snapshot", true)
if err != nil {
return nil, fmt.Errorf("failed to connect LevelDB freezer: %s", err)
}
return edb, nil
func NewEthDB(con *EthDBConfig) (ethdb.Database, error) {
return rawdb.Open(rawdb.OpenOptions{
Directory: con.DBPath,
AncientsDirectory: con.AncientDBPath,
Namespace: "ipld-eth-state-snapshot",
Cache: 1024,
Handles: 256,
ReadOnly: true,
})
}
// NewSnapshotService creates Service.

10
pkg/snapshot/service_test.go
View File

@ -35,10 +35,10 @@ func init() {
rand.Seed(time.Now().UnixNano())
}
func testConfig(leveldbpath, ancientdbpath string) *Config {
func testConfig(ethdbpath, ancientdbpath string) *Config {
return &Config{
Eth: &EthConfig{
LevelDBPath: leveldbpath,
Eth: &EthDBConfig{
DBPath: ethdbpath,
AncientDBPath: ancientdbpath,
NodeInfo: DefaultNodeInfo,
},
@ -210,7 +210,7 @@ func (expected selectiveData) verify(t *testing.T, data mocks.IndexerData) {
func doSnapshot(t *testing.T, chain *chaindata.Paths, params SnapshotParams) mocks.IndexerData {
chainDataPath, ancientDataPath := chain.ChainData, chain.Ancient
config := testConfig(chainDataPath, ancientDataPath)
edb, err := NewLevelDB(config.Eth)
edb, err := NewEthDB(config.Eth)
require.NoError(t, err)
defer edb.Close()
@ -232,7 +232,7 @@ func doSnapshotWithRecovery(
) mocks.IndexerData {
chainDataPath, ancientDataPath := chain.ChainData, chain.Ancient
config := testConfig(chainDataPath, ancientDataPath)
edb, err := NewLevelDB(config.Eth)
edb, err := NewEthDB(config.Eth)
require.NoError(t, err)
defer edb.Close()

6
scripts/compare-snapshots.sh
View File

@ -2,13 +2,13 @@
# Compare the full snapshot output from two versions of the service
#
# Usage: compare-versions.sh [-d <output-dir>] <binary-A> <binary-B>
#
# Configure the input data using environment vars.
(
set -u
: $SNAPSHOT_BLOCK_HEIGHT
: $LEVELDB_PATH
: $LEVELDB_ANCIENT
: $ETHDB_PATH
: $ETHDB_ANCIENT
: $ETH_GENESIS_BLOCK
)