2024-08-05 13:17:00 +00:00
9 changed files with 55 additions and 56 deletions
--- a/.gitea/workflows/test.yml
+++ b/.gitea/workflows/test.yml
@ -43,8 +43,8 @@ jobs:
        env:
          SNAPSHOT_MODE: postgres
          SNAPSHOT_BLOCK_HEIGHT: 32
-          LEVELDB_PATH: ./fixtures/chaindata/_data/small
-          LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient
+          ETHDB_PATH: ./fixtures/chaindata/_data/small
+          ETHDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient
          ETH_GENESIS_BLOCK: "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
        run: |
          until
@ -102,8 +102,8 @@ jobs:
      - name: Compare snapshot output
        env:
          SNAPSHOT_BLOCK_HEIGHT: 200
-          LEVELDB_PATH: ./fixtures/chaindata/_data/small2
-          LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
+          ETHDB_PATH: ./fixtures/chaindata/_data/small2
+          ETHDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
          ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
        run: |
          until
--- a/README.md
+++ b/README.md
@ -1,6 +1,6 @@
 # ipld-eth-state-snapshot

-> Tool for extracting the entire Ethereum state at a particular block height from leveldb into Postgres-backed IPFS
+> Tool for extracting the entire Ethereum state at a particular block height from a cold database into Postgres-backed IPFS

 [![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipld-eth-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/ipld-eth-state-snapshot)

@ -20,15 +20,15 @@ Config format:
 [snapshot]
    mode         = "file"           # indicates output mode <postgres | file>
    workers      = 4                # degree of concurrency: the state trie is subdivided into sections that are traversed and processed concurrently
-    blockHeight  = -1               # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in leveldb)
+    blockHeight  = -1               # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in ethdb)
    recoveryFile = "recovery_file"  # specifies a file to output recovery information on error or premature closure
    accounts = []                   # list of accounts (addresses) to take the snapshot for # SNAPSHOT_ACCOUNTS

-[leveldb]
-    # path to geth leveldb
-    path    = "/Users/user/Library/Ethereum/geth/chaindata"         # LEVELDB_PATH
+[ethdb]
+    # path to geth ethdb
+    path    = "/Users/user/Library/Ethereum/geth/chaindata"         # ETHDB_PATH
    # path to geth ancient database
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT
+    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # ETHDB_ANCIENT

 [database]
    # when operating in 'postgres' output mode
@ -66,14 +66,14 @@ Config format:
 ```

 > **Note:** previous versions of this service used different variable names. To update, change the following:
-> * `LVL_DB_PATH` => `LEVELDB_PATH`
-> * `ANCIENT_DB_PATH` => `LEVELDB_ANCIENT`
+> * `LVL_DB_PATH`, `LEVELDB_PATH` => `ETHDB_PATH`
+> * `ANCIENT_DB_PATH`, `LEVELDB_ANCIENT` => `ETHDB_ANCIENT`
 > * `LOGRUS_LEVEL`, `LOGRUS_FILE` => `LOG_LEVEL`, `LOG_FILE`, etc.


 ## Usage

-* For state snapshot from LevelDB:
+* For state snapshot from EthDB:

    ```bash
    ./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}
--- a/cmd/stateSnapshot.go
+++ b/cmd/stateSnapshot.go
@ -30,7 +30,7 @@ import (
 // stateSnapshotCmd represents the stateSnapshot command
 var stateSnapshotCmd = &cobra.Command{
 	Use:   "stateSnapshot",
-	Short: "Extract the entire Ethereum state from leveldb and publish into PG-IPFS",
+	Short: "Extract the entire Ethereum state from Ethdb and publish into PG-IPFS",
 	Long: `Usage

 ./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}`,
@ -47,9 +47,9 @@ func stateSnapshot() {
 	if err != nil {
 		logWithCommand.Fatalf("unable to initialize config: %v", err)
 	}
-	logWithCommand.Infof("opening levelDB and ancient data at %s and %s",
-		config.Eth.LevelDBPath, config.Eth.AncientDBPath)
-	edb, err := snapshot.NewLevelDB(config.Eth)
+	logWithCommand.Infof("opening ethdb and ancient data at %s and %s",
+		config.Eth.DBPath, config.Eth.AncientDBPath)
+	edb, err := snapshot.NewEthDB(config.Eth)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@ -99,8 +99,8 @@ func stateSnapshot() {
 func init() {
 	rootCmd.AddCommand(stateSnapshotCmd)

-	stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_PATH_CLI, "", "path to primary datastore")
-	stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_ANCIENT_CLI, "", "path to ancient datastore")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.ETHDB_PATH_CLI, "", "path to primary datastore")
+	stateSnapshotCmd.PersistentFlags().String(snapshot.ETHDB_ANCIENT_CLI, "", "path to ancient datastore")
 	stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI, "", "block height to extract state at")
 	stateSnapshotCmd.PersistentFlags().Int(snapshot.SNAPSHOT_WORKERS_CLI, 1, "number of concurrent workers to use")
 	stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_RECOVERY_FILE_CLI, "", "file to recover from a previous iteration")
@ -108,8 +108,8 @@ func init() {
 	stateSnapshotCmd.PersistentFlags().String(snapshot.FILE_OUTPUT_DIR_CLI, "", "directory for writing ouput to while operating in 'file' mode")
 	stateSnapshotCmd.PersistentFlags().StringArray(snapshot.SNAPSHOT_ACCOUNTS_CLI, nil, "list of account addresses to limit snapshot to")

-	viper.BindPFlag(snapshot.LEVELDB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_PATH_CLI))
-	viper.BindPFlag(snapshot.LEVELDB_ANCIENT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_ANCIENT_CLI))
+	viper.BindPFlag(snapshot.ETHDB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ETHDB_PATH_CLI))
+	viper.BindPFlag(snapshot.ETHDB_ANCIENT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ETHDB_ANCIENT_CLI))
 	viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI))
 	viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_WORKERS_CLI))
 	viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_RECOVERY_FILE_CLI))
--- a/environments/example.toml
+++ b/environments/example.toml
@ -5,7 +5,7 @@
    user     = "vdbm"
    password = "password"

-[leveldb]
+[ethdb]
    path = "/Users/user/go/src/github.com/cerc-io/ipld-eth-state-snapshot/fixture/chain2data"
    ancient = "/Users/user/go/src/github.com/cerc-io/ipld-eth-state-snapshot/fixture/chain2data/ancient"

--- a/pkg/snapshot/config.go
+++ b/pkg/snapshot/config.go
@ -41,15 +41,15 @@ const (

 // Config contains params for both databases the service uses
 type Config struct {
-	Eth     *EthConfig
+	Eth     *EthDBConfig
 	DB      *DBConfig
 	File    *FileConfig
 	Service *ServiceConfig
 }

-// EthConfig is config parameters for the chain.
-type EthConfig struct {
-	LevelDBPath   string
+// EthDBConfig is config parameters for the chain DB.
+type EthDBConfig struct {
+	DBPath        string
 	AncientDBPath string
 	NodeInfo      ethNode.Info
 }
@ -67,7 +67,7 @@ type ServiceConfig struct {

 func NewConfig(mode SnapshotMode) (*Config, error) {
 	ret := &Config{
-		&EthConfig{},
+		&EthDBConfig{},
 		&DBConfig{},
 		&FileConfig{},
 		&ServiceConfig{},
@ -77,7 +77,7 @@ func NewConfig(mode SnapshotMode) (*Config, error) {

 func NewInPlaceSnapshotConfig() *Config {
 	ret := &Config{
-		&EthConfig{},
+		&EthDBConfig{},
 		&DBConfig{},
 		&FileConfig{},
 		&ServiceConfig{},
@ -104,11 +104,11 @@ func (c *Config) Init(mode SnapshotMode) error {
 		ChainID:      viper.GetUint64(ETH_CHAIN_ID_TOML),
 	}

-	viper.BindEnv(LEVELDB_ANCIENT_TOML, LEVELDB_ANCIENT)
-	viper.BindEnv(LEVELDB_PATH_TOML, LEVELDB_PATH)
+	viper.BindEnv(ETHDB_ANCIENT_TOML, ETHDB_ANCIENT)
+	viper.BindEnv(ETHDB_PATH_TOML, ETHDB_PATH)

-	c.Eth.AncientDBPath = viper.GetString(LEVELDB_ANCIENT_TOML)
-	c.Eth.LevelDBPath = viper.GetString(LEVELDB_PATH_TOML)
+	c.Eth.AncientDBPath = viper.GetString(ETHDB_ANCIENT_TOML)
+	c.Eth.DBPath = viper.GetString(ETHDB_PATH_TOML)

 	switch mode {
 	case FileSnapshot:
--- a/pkg/snapshot/env.go
+++ b/pkg/snapshot/env.go
@ -34,8 +34,8 @@ const (

 	FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"

-	LEVELDB_ANCIENT = "LEVELDB_ANCIENT"
-	LEVELDB_PATH    = "LEVELDB_PATH"
+	ETHDB_ANCIENT = "ETHDB_ANCIENT"
+	ETHDB_PATH    = "ETHDB_PATH"

 	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
 	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
@ -72,8 +72,8 @@ const (

 	FILE_OUTPUT_DIR_TOML = "file.outputDir"

-	LEVELDB_ANCIENT_TOML = "leveldb.ancient"
-	LEVELDB_PATH_TOML    = "leveldb.path"
+	ETHDB_ANCIENT_TOML = "ethdb.ancient"
+	ETHDB_PATH_TOML    = "ethdb.path"

 	ETH_CLIENT_NAME_TOML   = "ethereum.clientName"
 	ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
@ -110,8 +110,8 @@ const (

 	FILE_OUTPUT_DIR_CLI = "output-dir"

-	LEVELDB_ANCIENT_CLI = "ancient-path"
-	LEVELDB_PATH_CLI    = "leveldb-path"
+	ETHDB_ANCIENT_CLI = "ancient-path"
+	ETHDB_PATH_CLI    = "ethdb-path"

 	ETH_CLIENT_NAME_CLI   = "ethereum-client-name"
 	ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block"
--- a/pkg/snapshot/service.go
+++ b/pkg/snapshot/service.go
@ -56,16 +56,15 @@ type Service struct {
 	recoveryFile string
 }

-func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
-	kvdb, err := rawdb.NewLevelDBDatabase(con.LevelDBPath, 1024, 256, "ipld-eth-state-snapshot", true)
-	if err != nil {
-		return nil, fmt.Errorf("failed to connect LevelDB: %s", err)
-	}
-	edb, err := rawdb.NewDatabaseWithFreezer(kvdb, con.AncientDBPath, "ipld-eth-state-snapshot", true)
-	if err != nil {
-		return nil, fmt.Errorf("failed to connect LevelDB freezer: %s", err)
-	}
-	return edb, nil
+func NewEthDB(con *EthDBConfig) (ethdb.Database, error) {
+	return rawdb.Open(rawdb.OpenOptions{
+		Directory:         con.DBPath,
+		AncientsDirectory: con.AncientDBPath,
+		Namespace:         "ipld-eth-state-snapshot",
+		Cache:             1024,
+		Handles:           256,
+		ReadOnly:          true,
+	})
 }

 // NewSnapshotService creates Service.
--- a/pkg/snapshot/service_test.go
+++ b/pkg/snapshot/service_test.go
@ -35,10 +35,10 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 }

-func testConfig(leveldbpath, ancientdbpath string) *Config {
+func testConfig(ethdbpath, ancientdbpath string) *Config {
 	return &Config{
-		Eth: &EthConfig{
-			LevelDBPath:   leveldbpath,
+		Eth: &EthDBConfig{
+			DBPath:        ethdbpath,
 			AncientDBPath: ancientdbpath,
 			NodeInfo:      DefaultNodeInfo,
 		},
@ -210,7 +210,7 @@ func (expected selectiveData) verify(t *testing.T, data mocks.IndexerData) {
 func doSnapshot(t *testing.T, chain *chaindata.Paths, params SnapshotParams) mocks.IndexerData {
 	chainDataPath, ancientDataPath := chain.ChainData, chain.Ancient
 	config := testConfig(chainDataPath, ancientDataPath)
-	edb, err := NewLevelDB(config.Eth)
+	edb, err := NewEthDB(config.Eth)
 	require.NoError(t, err)
 	defer edb.Close()

@ -232,7 +232,7 @@ func doSnapshotWithRecovery(
 ) mocks.IndexerData {
 	chainDataPath, ancientDataPath := chain.ChainData, chain.Ancient
 	config := testConfig(chainDataPath, ancientDataPath)
-	edb, err := NewLevelDB(config.Eth)
+	edb, err := NewEthDB(config.Eth)
 	require.NoError(t, err)
 	defer edb.Close()

--- a/scripts/compare-snapshots.sh
+++ b/scripts/compare-snapshots.sh
@ -2,13 +2,13 @@
 # Compare the full snapshot output from two versions of the service
 #
 # Usage: compare-versions.sh [-d <output-dir>] <binary-A> <binary-B>
-#
+
 # Configure the input data using environment vars.
 (
  set -u
  : $SNAPSHOT_BLOCK_HEIGHT
-  : $LEVELDB_PATH
-  : $LEVELDB_ANCIENT
+  : $ETHDB_PATH
+  : $ETHDB_ANCIENT
  : $ETH_GENESIS_BLOCK
 )