support pebble db

.gitea/workflows/test.yml

@@ -43,8 +43,8 @@ jobs:
         env:
           SNAPSHOT_MODE: postgres
           SNAPSHOT_BLOCK_HEIGHT: 32
-          LEVELDB_PATH: ./fixtures/chaindata/_data/small
+          ETHDB_PATH: ./fixtures/chaindata/_data/small
-          LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient
+          ETHDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient
           ETH_GENESIS_BLOCK: "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
         run: |
           until
@@ -102,8 +102,8 @@ jobs:
       - name: Compare snapshot output
         env:
           SNAPSHOT_BLOCK_HEIGHT: 200
-          LEVELDB_PATH: ./fixtures/chaindata/_data/small2
+          ETHDB_PATH: ./fixtures/chaindata/_data/small2
-          LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
+          ETHDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
           ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
         run: |
           until

README.md

@@ -1,6 +1,6 @@
 # ipld-eth-state-snapshot
 
-> Tool for extracting the entire Ethereum state at a particular block height from leveldb into Postgres-backed IPFS
+> Tool for extracting the entire Ethereum state at a particular block height from a cold database into Postgres-backed IPFS
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipld-eth-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/ipld-eth-state-snapshot)
 
@@ -20,15 +20,15 @@ Config format:
 [snapshot]
     mode         = "file"           # indicates output mode <postgres | file>
     workers      = 4                # degree of concurrency: the state trie is subdivided into sections that are traversed and processed concurrently
-    blockHeight  = -1               # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in leveldb)
+    blockHeight  = -1               # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in ethdb)
     recoveryFile = "recovery_file"  # specifies a file to output recovery information on error or premature closure
     accounts = []                   # list of accounts (addresses) to take the snapshot for # SNAPSHOT_ACCOUNTS
 
-[leveldb]
+[ethdb]
-    # path to geth leveldb
+    # path to geth ethdb
-    path    = "/Users/user/Library/Ethereum/geth/chaindata"         # LEVELDB_PATH
+    path    = "/Users/user/Library/Ethereum/geth/chaindata"         # ETHDB_PATH
     # path to geth ancient database
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT
+    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # ETHDB_ANCIENT
 
 [database]
     # when operating in 'postgres' output mode
@@ -66,14 +66,14 @@ Config format:
 ```
 
 > **Note:** previous versions of this service used different variable names. To update, change the following:
-> * `LVL_DB_PATH` => `LEVELDB_PATH`
+> * `LVL_DB_PATH`, `LEVELDB_PATH` => `ETHDB_PATH`
-> * `ANCIENT_DB_PATH` => `LEVELDB_ANCIENT`
+> * `ANCIENT_DB_PATH`, `LEVELDB_ANCIENT` => `ETHDB_ANCIENT`
 > * `LOGRUS_LEVEL`, `LOGRUS_FILE` => `LOG_LEVEL`, `LOG_FILE`, etc.
 
 
 ## Usage
 
-* For state snapshot from LevelDB:
+* For state snapshot from EthDB:
 
     ```bash
     ./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}

cmd/stateSnapshot.go

@@ -30,7 +30,7 @@ import (
 // stateSnapshotCmd represents the stateSnapshot command
 var stateSnapshotCmd = &cobra.Command{
 	Use:   "stateSnapshot",
-	Short: "Extract the entire Ethereum state from leveldb and publish into PG-IPFS",
+	Short: "Extract the entire Ethereum state from Ethdb and publish into PG-IPFS",
 	Long: `Usage
 
 ./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}`,
@@ -47,9 +47,9 @@ func stateSnapshot() {
 	if err != nil {
 		logWithCommand.Fatalf("unable to initialize config: %v", err)
 	}
-	logWithCommand.Infof("opening levelDB and ancient data at %s and %s",
+	logWithCommand.Infof("opening ethdb and ancient data at %s and %s",
-		config.Eth.LevelDBPath, config.Eth.AncientDBPath)
+		config.Eth.DBPath, config.Eth.AncientDBPath)
-	edb, err := snapshot.NewLevelDB(config.Eth)
+	edb, err := snapshot.NewEthDB(config.Eth)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -99,8 +99,8 @@ func stateSnapshot() {
 func init() {
 	rootCmd.AddCommand(stateSnapshotCmd)
 
-	stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_PATH_CLI, "", "path to primary datastore")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.ETHDB_PATH_CLI, "", "path to primary datastore")
-	stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_ANCIENT_CLI, "", "path to ancient datastore")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.ETHDB_ANCIENT_CLI, "", "path to ancient datastore")
 	stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI, "", "block height to extract state at")
 	stateSnapshotCmd.PersistentFlags().Int(snapshot.SNAPSHOT_WORKERS_CLI, 1, "number of concurrent workers to use")
 	stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_RECOVERY_FILE_CLI, "", "file to recover from a previous iteration")
@@ -108,8 +108,8 @@ func init() {
 	stateSnapshotCmd.PersistentFlags().String(snapshot.FILE_OUTPUT_DIR_CLI, "", "directory for writing ouput to while operating in 'file' mode")
 	stateSnapshotCmd.PersistentFlags().StringArray(snapshot.SNAPSHOT_ACCOUNTS_CLI, nil, "list of account addresses to limit snapshot to")
 
-	viper.BindPFlag(snapshot.LEVELDB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_PATH_CLI))
+	viper.BindPFlag(snapshot.ETHDB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ETHDB_PATH_CLI))
-	viper.BindPFlag(snapshot.LEVELDB_ANCIENT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_ANCIENT_CLI))
+	viper.BindPFlag(snapshot.ETHDB_ANCIENT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ETHDB_ANCIENT_CLI))
 	viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI))
 	viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_WORKERS_CLI))
 	viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_RECOVERY_FILE_CLI))

environments/example.toml

@@ -5,7 +5,7 @@
     user     = "vdbm"
     password = "password"
 
-[leveldb]
+[ethdb]
     path = "/Users/user/go/src/github.com/cerc-io/ipld-eth-state-snapshot/fixture/chain2data"
     ancient = "/Users/user/go/src/github.com/cerc-io/ipld-eth-state-snapshot/fixture/chain2data/ancient"
 

pkg/snapshot/config.go

@@ -41,15 +41,15 @@ const (
 
 // Config contains params for both databases the service uses
 type Config struct {
-	Eth     *EthConfig
+	Eth     *EthDBConfig
 	DB      *DBConfig
 	File    *FileConfig
 	Service *ServiceConfig
 }
 
-// EthConfig is config parameters for the chain.
+// EthDBConfig is config parameters for the chain DB.
-type EthConfig struct {
+type EthDBConfig struct {
-	LevelDBPath   string
+	DBPath        string
 	AncientDBPath string
 	NodeInfo      ethNode.Info
 }
@@ -67,7 +67,7 @@ type ServiceConfig struct {
 
 func NewConfig(mode SnapshotMode) (*Config, error) {
 	ret := &Config{
-		&EthConfig{},
+		&EthDBConfig{},
 		&DBConfig{},
 		&FileConfig{},
 		&ServiceConfig{},
@@ -77,7 +77,7 @@ func NewConfig(mode SnapshotMode) (*Config, error) {
 
 func NewInPlaceSnapshotConfig() *Config {
 	ret := &Config{
-		&EthConfig{},
+		&EthDBConfig{},
 		&DBConfig{},
 		&FileConfig{},
 		&ServiceConfig{},
@@ -104,11 +104,11 @@ func (c *Config) Init(mode SnapshotMode) error {
 		ChainID:      viper.GetUint64(ETH_CHAIN_ID_TOML),
 	}
 
-	viper.BindEnv(LEVELDB_ANCIENT_TOML, LEVELDB_ANCIENT)
+	viper.BindEnv(ETHDB_ANCIENT_TOML, ETHDB_ANCIENT)
-	viper.BindEnv(LEVELDB_PATH_TOML, LEVELDB_PATH)
+	viper.BindEnv(ETHDB_PATH_TOML, ETHDB_PATH)
 
-	c.Eth.AncientDBPath = viper.GetString(LEVELDB_ANCIENT_TOML)
+	c.Eth.AncientDBPath = viper.GetString(ETHDB_ANCIENT_TOML)
-	c.Eth.LevelDBPath = viper.GetString(LEVELDB_PATH_TOML)
+	c.Eth.DBPath = viper.GetString(ETHDB_PATH_TOML)
 
 	switch mode {
 	case FileSnapshot:

pkg/snapshot/env.go

@@ -34,8 +34,8 @@ const (
 
 	FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"
 
-	LEVELDB_ANCIENT = "LEVELDB_ANCIENT"
+	ETHDB_ANCIENT = "ETHDB_ANCIENT"
-	LEVELDB_PATH    = "LEVELDB_PATH"
+	ETHDB_PATH    = "ETHDB_PATH"
 
 	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
 	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
@@ -72,8 +72,8 @@ const (
 
 	FILE_OUTPUT_DIR_TOML = "file.outputDir"
 
-	LEVELDB_ANCIENT_TOML = "leveldb.ancient"
+	ETHDB_ANCIENT_TOML = "ethdb.ancient"
-	LEVELDB_PATH_TOML    = "leveldb.path"
+	ETHDB_PATH_TOML    = "ethdb.path"
 
 	ETH_CLIENT_NAME_TOML   = "ethereum.clientName"
 	ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
@@ -110,8 +110,8 @@ const (
 
 	FILE_OUTPUT_DIR_CLI = "output-dir"
 
-	LEVELDB_ANCIENT_CLI = "ancient-path"
+	ETHDB_ANCIENT_CLI = "ancient-path"
-	LEVELDB_PATH_CLI    = "leveldb-path"
+	ETHDB_PATH_CLI    = "ethdb-path"
 
 	ETH_CLIENT_NAME_CLI   = "ethereum-client-name"
 	ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block"

pkg/snapshot/service.go

@@ -56,16 +56,15 @@ type Service struct {
 	recoveryFile string
 }
 
-func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
+func NewEthDB(con *EthDBConfig) (ethdb.Database, error) {
-	kvdb, err := rawdb.NewLevelDBDatabase(con.LevelDBPath, 1024, 256, "ipld-eth-state-snapshot", true)
+	return rawdb.Open(rawdb.OpenOptions{
-	if err != nil {
+		Directory:         con.DBPath,
-		return nil, fmt.Errorf("failed to connect LevelDB: %s", err)
+		AncientsDirectory: con.AncientDBPath,
-	}
+		Namespace:         "ipld-eth-state-snapshot",
-	edb, err := rawdb.NewDatabaseWithFreezer(kvdb, con.AncientDBPath, "ipld-eth-state-snapshot", true)
+		Cache:             1024,
-	if err != nil {
+		Handles:           256,
-		return nil, fmt.Errorf("failed to connect LevelDB freezer: %s", err)
+		ReadOnly:          true,
-	}
+	})
-	return edb, nil
 }
 
 // NewSnapshotService creates Service.

pkg/snapshot/service_test.go

@@ -35,10 +35,10 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
-func testConfig(leveldbpath, ancientdbpath string) *Config {
+func testConfig(ethdbpath, ancientdbpath string) *Config {
 	return &Config{
-		Eth: &EthConfig{
+		Eth: &EthDBConfig{
-			LevelDBPath:   leveldbpath,
+			DBPath:        ethdbpath,
 			AncientDBPath: ancientdbpath,
 			NodeInfo:      DefaultNodeInfo,
 		},
@@ -210,7 +210,7 @@ func (expected selectiveData) verify(t *testing.T, data mocks.IndexerData) {
 func doSnapshot(t *testing.T, chain *chaindata.Paths, params SnapshotParams) mocks.IndexerData {
 	chainDataPath, ancientDataPath := chain.ChainData, chain.Ancient
 	config := testConfig(chainDataPath, ancientDataPath)
-	edb, err := NewLevelDB(config.Eth)
+	edb, err := NewEthDB(config.Eth)
 	require.NoError(t, err)
 	defer edb.Close()
 
@@ -232,7 +232,7 @@ func doSnapshotWithRecovery(
 ) mocks.IndexerData {
 	chainDataPath, ancientDataPath := chain.ChainData, chain.Ancient
 	config := testConfig(chainDataPath, ancientDataPath)
-	edb, err := NewLevelDB(config.Eth)
+	edb, err := NewEthDB(config.Eth)
 	require.NoError(t, err)
 	defer edb.Close()
 

scripts/compare-snapshots.sh

@@ -2,13 +2,13 @@
 # Compare the full snapshot output from two versions of the service
 #
 # Usage: compare-versions.sh [-d <output-dir>] <binary-A> <binary-B>
-#
+
 # Configure the input data using environment vars.
 (
   set -u
   : $SNAPSHOT_BLOCK_HEIGHT
-  : $LEVELDB_PATH
+  : $ETHDB_PATH
-  : $LEVELDB_ANCIENT
+  : $ETHDB_ANCIENT
   : $ETH_GENESIS_BLOCK
 )