diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index ae30fdf..00d4797 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -43,8 +43,8 @@ jobs: env: SNAPSHOT_MODE: postgres SNAPSHOT_BLOCK_HEIGHT: 32 - LEVELDB_PATH: ./fixtures/chaindata/_data/small - LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient + ETHDB_PATH: ./fixtures/chaindata/_data/small + ETHDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient ETH_GENESIS_BLOCK: "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05" run: | until @@ -102,8 +102,8 @@ jobs: - name: Compare snapshot output env: SNAPSHOT_BLOCK_HEIGHT: 200 - LEVELDB_PATH: ./fixtures/chaindata/_data/small2 - LEVELDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient + ETHDB_PATH: ./fixtures/chaindata/_data/small2 + ETHDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614" run: | until diff --git a/README.md b/README.md index 7b4ceb7..2614ba1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ipld-eth-state-snapshot -> Tool for extracting the entire Ethereum state at a particular block height from leveldb into Postgres-backed IPFS +> Tool for extracting the entire Ethereum state at a particular block height from a cold database into Postgres-backed IPFS [![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipld-eth-state-snapshot)](https://goreportcard.com/report/github.com/vulcanize/ipld-eth-state-snapshot) @@ -20,15 +20,15 @@ Config format: [snapshot] mode = "file" # indicates output mode workers = 4 # degree of concurrency: the state trie is subdivided into sections that are traversed and processed concurrently - blockHeight = -1 # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in leveldb) + blockHeight = -1 # blockheight to perform the snapshot at (-1 indicates to use the latest blockheight found in ethdb) recoveryFile = "recovery_file" # specifies a file to output recovery information on error or premature closure accounts = [] # list of accounts (addresses) to take the snapshot for # SNAPSHOT_ACCOUNTS -[leveldb] - # path to geth leveldb - path = "/Users/user/Library/Ethereum/geth/chaindata" # LEVELDB_PATH +[ethdb] + # path to geth ethdb + path = "/Users/user/Library/Ethereum/geth/chaindata" # ETHDB_PATH # path to geth ancient database - ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT + ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # ETHDB_ANCIENT [database] # when operating in 'postgres' output mode @@ -66,14 +66,14 @@ Config format: ``` > **Note:** previous versions of this service used different variable names. To update, change the following: -> * `LVL_DB_PATH` => `LEVELDB_PATH` -> * `ANCIENT_DB_PATH` => `LEVELDB_ANCIENT` +> * `LVL_DB_PATH`, `LEVELDB_PATH` => `ETHDB_PATH` +> * `ANCIENT_DB_PATH`, `LEVELDB_ANCIENT` => `ETHDB_ANCIENT` > * `LOGRUS_LEVEL`, `LOGRUS_FILE` => `LOG_LEVEL`, `LOG_FILE`, etc. ## Usage -* For state snapshot from LevelDB: +* For state snapshot from EthDB: ```bash ./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file} diff --git a/cmd/stateSnapshot.go b/cmd/stateSnapshot.go index ef7ac45..2b870db 100644 --- a/cmd/stateSnapshot.go +++ b/cmd/stateSnapshot.go @@ -30,7 +30,7 @@ import ( // stateSnapshotCmd represents the stateSnapshot command var stateSnapshotCmd = &cobra.Command{ Use: "stateSnapshot", - Short: "Extract the entire Ethereum state from leveldb and publish into PG-IPFS", + Short: "Extract the entire Ethereum state from Ethdb and publish into PG-IPFS", Long: `Usage ./ipld-eth-state-snapshot stateSnapshot --config={path to toml config file}`, @@ -47,9 +47,9 @@ func stateSnapshot() { if err != nil { logWithCommand.Fatalf("unable to initialize config: %v", err) } - logWithCommand.Infof("opening levelDB and ancient data at %s and %s", - config.Eth.LevelDBPath, config.Eth.AncientDBPath) - edb, err := snapshot.NewLevelDB(config.Eth) + logWithCommand.Infof("opening ethdb and ancient data at %s and %s", + config.Eth.DBPath, config.Eth.AncientDBPath) + edb, err := snapshot.NewEthDB(config.Eth) if err != nil { logWithCommand.Fatal(err) } @@ -99,8 +99,8 @@ func stateSnapshot() { func init() { rootCmd.AddCommand(stateSnapshotCmd) - stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_PATH_CLI, "", "path to primary datastore") - stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_ANCIENT_CLI, "", "path to ancient datastore") + stateSnapshotCmd.PersistentFlags().String(snapshot.ETHDB_PATH_CLI, "", "path to primary datastore") + stateSnapshotCmd.PersistentFlags().String(snapshot.ETHDB_ANCIENT_CLI, "", "path to ancient datastore") stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI, "", "block height to extract state at") stateSnapshotCmd.PersistentFlags().Int(snapshot.SNAPSHOT_WORKERS_CLI, 1, "number of concurrent workers to use") stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_RECOVERY_FILE_CLI, "", "file to recover from a previous iteration") @@ -108,8 +108,8 @@ func init() { stateSnapshotCmd.PersistentFlags().String(snapshot.FILE_OUTPUT_DIR_CLI, "", "directory for writing ouput to while operating in 'file' mode") stateSnapshotCmd.PersistentFlags().StringArray(snapshot.SNAPSHOT_ACCOUNTS_CLI, nil, "list of account addresses to limit snapshot to") - viper.BindPFlag(snapshot.LEVELDB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_PATH_CLI)) - viper.BindPFlag(snapshot.LEVELDB_ANCIENT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_ANCIENT_CLI)) + viper.BindPFlag(snapshot.ETHDB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ETHDB_PATH_CLI)) + viper.BindPFlag(snapshot.ETHDB_ANCIENT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ETHDB_ANCIENT_CLI)) viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI)) viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_WORKERS_CLI)) viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_RECOVERY_FILE_CLI)) diff --git a/environments/example.toml b/environments/example.toml index 91f4858..ab0d1f3 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -5,7 +5,7 @@ user = "vdbm" password = "password" -[leveldb] +[ethdb] path = "/Users/user/go/src/github.com/cerc-io/ipld-eth-state-snapshot/fixture/chain2data" ancient = "/Users/user/go/src/github.com/cerc-io/ipld-eth-state-snapshot/fixture/chain2data/ancient" diff --git a/pkg/snapshot/config.go b/pkg/snapshot/config.go index 8933868..ba7a52e 100644 --- a/pkg/snapshot/config.go +++ b/pkg/snapshot/config.go @@ -41,15 +41,15 @@ const ( // Config contains params for both databases the service uses type Config struct { - Eth *EthConfig + Eth *EthDBConfig DB *DBConfig File *FileConfig Service *ServiceConfig } -// EthConfig is config parameters for the chain. -type EthConfig struct { - LevelDBPath string +// EthDBConfig is config parameters for the chain DB. +type EthDBConfig struct { + DBPath string AncientDBPath string NodeInfo ethNode.Info } @@ -67,7 +67,7 @@ type ServiceConfig struct { func NewConfig(mode SnapshotMode) (*Config, error) { ret := &Config{ - &EthConfig{}, + &EthDBConfig{}, &DBConfig{}, &FileConfig{}, &ServiceConfig{}, @@ -77,7 +77,7 @@ func NewConfig(mode SnapshotMode) (*Config, error) { func NewInPlaceSnapshotConfig() *Config { ret := &Config{ - &EthConfig{}, + &EthDBConfig{}, &DBConfig{}, &FileConfig{}, &ServiceConfig{}, @@ -104,11 +104,11 @@ func (c *Config) Init(mode SnapshotMode) error { ChainID: viper.GetUint64(ETH_CHAIN_ID_TOML), } - viper.BindEnv(LEVELDB_ANCIENT_TOML, LEVELDB_ANCIENT) - viper.BindEnv(LEVELDB_PATH_TOML, LEVELDB_PATH) + viper.BindEnv(ETHDB_ANCIENT_TOML, ETHDB_ANCIENT) + viper.BindEnv(ETHDB_PATH_TOML, ETHDB_PATH) - c.Eth.AncientDBPath = viper.GetString(LEVELDB_ANCIENT_TOML) - c.Eth.LevelDBPath = viper.GetString(LEVELDB_PATH_TOML) + c.Eth.AncientDBPath = viper.GetString(ETHDB_ANCIENT_TOML) + c.Eth.DBPath = viper.GetString(ETHDB_PATH_TOML) switch mode { case FileSnapshot: diff --git a/pkg/snapshot/env.go b/pkg/snapshot/env.go index 5ac202d..378f772 100644 --- a/pkg/snapshot/env.go +++ b/pkg/snapshot/env.go @@ -34,8 +34,8 @@ const ( FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR" - LEVELDB_ANCIENT = "LEVELDB_ANCIENT" - LEVELDB_PATH = "LEVELDB_PATH" + ETHDB_ANCIENT = "ETHDB_ANCIENT" + ETHDB_PATH = "ETHDB_PATH" ETH_CLIENT_NAME = "ETH_CLIENT_NAME" ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK" @@ -72,8 +72,8 @@ const ( FILE_OUTPUT_DIR_TOML = "file.outputDir" - LEVELDB_ANCIENT_TOML = "leveldb.ancient" - LEVELDB_PATH_TOML = "leveldb.path" + ETHDB_ANCIENT_TOML = "ethdb.ancient" + ETHDB_PATH_TOML = "ethdb.path" ETH_CLIENT_NAME_TOML = "ethereum.clientName" ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock" @@ -110,8 +110,8 @@ const ( FILE_OUTPUT_DIR_CLI = "output-dir" - LEVELDB_ANCIENT_CLI = "ancient-path" - LEVELDB_PATH_CLI = "leveldb-path" + ETHDB_ANCIENT_CLI = "ancient-path" + ETHDB_PATH_CLI = "ethdb-path" ETH_CLIENT_NAME_CLI = "ethereum-client-name" ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block" diff --git a/pkg/snapshot/service.go b/pkg/snapshot/service.go index e5404a8..7d871f0 100644 --- a/pkg/snapshot/service.go +++ b/pkg/snapshot/service.go @@ -56,16 +56,15 @@ type Service struct { recoveryFile string } -func NewLevelDB(con *EthConfig) (ethdb.Database, error) { - kvdb, err := rawdb.NewLevelDBDatabase(con.LevelDBPath, 1024, 256, "ipld-eth-state-snapshot", true) - if err != nil { - return nil, fmt.Errorf("failed to connect LevelDB: %s", err) - } - edb, err := rawdb.NewDatabaseWithFreezer(kvdb, con.AncientDBPath, "ipld-eth-state-snapshot", true) - if err != nil { - return nil, fmt.Errorf("failed to connect LevelDB freezer: %s", err) - } - return edb, nil +func NewEthDB(con *EthDBConfig) (ethdb.Database, error) { + return rawdb.Open(rawdb.OpenOptions{ + Directory: con.DBPath, + AncientsDirectory: con.AncientDBPath, + Namespace: "ipld-eth-state-snapshot", + Cache: 1024, + Handles: 256, + ReadOnly: true, + }) } // NewSnapshotService creates Service. diff --git a/pkg/snapshot/service_test.go b/pkg/snapshot/service_test.go index 64b34db..5609431 100644 --- a/pkg/snapshot/service_test.go +++ b/pkg/snapshot/service_test.go @@ -35,10 +35,10 @@ func init() { rand.Seed(time.Now().UnixNano()) } -func testConfig(leveldbpath, ancientdbpath string) *Config { +func testConfig(ethdbpath, ancientdbpath string) *Config { return &Config{ - Eth: &EthConfig{ - LevelDBPath: leveldbpath, + Eth: &EthDBConfig{ + DBPath: ethdbpath, AncientDBPath: ancientdbpath, NodeInfo: DefaultNodeInfo, }, @@ -210,7 +210,7 @@ func (expected selectiveData) verify(t *testing.T, data mocks.IndexerData) { func doSnapshot(t *testing.T, chain *chaindata.Paths, params SnapshotParams) mocks.IndexerData { chainDataPath, ancientDataPath := chain.ChainData, chain.Ancient config := testConfig(chainDataPath, ancientDataPath) - edb, err := NewLevelDB(config.Eth) + edb, err := NewEthDB(config.Eth) require.NoError(t, err) defer edb.Close() @@ -232,7 +232,7 @@ func doSnapshotWithRecovery( ) mocks.IndexerData { chainDataPath, ancientDataPath := chain.ChainData, chain.Ancient config := testConfig(chainDataPath, ancientDataPath) - edb, err := NewLevelDB(config.Eth) + edb, err := NewEthDB(config.Eth) require.NoError(t, err) defer edb.Close() diff --git a/scripts/compare-snapshots.sh b/scripts/compare-snapshots.sh index 6ce0029..cdd762d 100755 --- a/scripts/compare-snapshots.sh +++ b/scripts/compare-snapshots.sh @@ -2,13 +2,13 @@ # Compare the full snapshot output from two versions of the service # # Usage: compare-versions.sh [-d ] -# + # Configure the input data using environment vars. ( set -u : $SNAPSHOT_BLOCK_HEIGHT - : $LEVELDB_PATH - : $LEVELDB_ANCIENT + : $ETHDB_PATH + : $ETHDB_ANCIENT : $ETH_GENESIS_BLOCK )