Merge pull request #110 from cerc-io/ian_util

latest block height util

README.md

@@ -297,3 +297,12 @@ An example config file:
     ```
 
 * NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported statediff data with the data generated in `postgres` mode. Reference: https://www.postgresql.org/docs/14/sql-copy.html
+
+### Stats
+
+The binary includes a `stats` command which reports stats for the offline or remote levelDB.
+
+At this time, the only stat supported is to return the latest/highest block height and hash found the levelDB, this is
+useful for determining what the upper limit is for a standalone statediffing process on a given levelDB.
+
+`./eth-statediff-service stats --config={path to toml config file}`

cmd/root.go

@@ -304,11 +304,11 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 	if err != nil {
 		return nil, err
 	}
-	logWithCommand.Infof("configuring service for database type: %s", dbType)
+	logWithCommand.Infof("Configuring service for database type: %s", dbType)
 	var indexerConfig interfaces.Config
 	switch dbType {
 	case shared.FILE:
-		logWithCommand.Info("starting in sql file writing mode")
+		logWithCommand.Info("Starting in sql file writing mode")
 
 		fileModeStr := viper.GetString("database.fileMode")
 		fileMode, err := file.ResolveFileMode(fileModeStr)
@@ -318,12 +318,12 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 
 		filePathStr := viper.GetString("database.filePath")
 		if fileMode == file.SQL && filePathStr == "" {
-			logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
+			logWithCommand.Fatal("When operating in sql file writing mode a file path must be provided")
 		}
 
 		fileCsvDirStr := viper.GetString("database.fileCsvDir")
 		if fileMode == file.CSV && fileCsvDirStr == "" {
-			logWithCommand.Fatal("when operating in csv file writing mode a directory path must be provided")
+			logWithCommand.Fatal("When operating in csv file writing mode a directory path must be provided")
 		}
 
 		indexerConfig = file.Config{
@@ -332,7 +332,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 			FilePath:  filePathStr,
 		}
 	case shared.DUMP:
-		logWithCommand.Info("starting in data dump mode")
+		logWithCommand.Info("Starting in data dump mode")
 		dumpDstStr := viper.GetString("database.dumpDestination")
 		dumpDst, err := dump.ResolveDumpType(dumpDstStr)
 		if err != nil {
@@ -349,7 +349,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 			return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
 		}
 	case shared.POSTGRES:
-		logWithCommand.Info("starting in postgres mode")
+		logWithCommand.Info("Starting in postgres mode")
 		driverTypeStr := viper.GetString("database.driver")
 		driverType, err := postgres.ResolveDriverType(driverTypeStr)
 		if err != nil {

cmd/serve.go

@@ -63,7 +63,19 @@ func serve() {
 	logWithCommand.Info("Running eth-statediff-service serve command")
 	logWithCommand.Infof("Parallelism: %d", maxParallelism())
 
-	statediffService, err := createStateDiffService()
+	reader, chainConf, nodeInfo := instantiateLevelDBReader()
+
+	// report latest block info
+	header, err := reader.GetLatestHeader()
+	if err != nil {
+		logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
+	}
+	if header.Number == nil {
+		logWithCommand.Fatal("Latest header found in levelDB has a nil block height")
+	}
+	logWithCommand.Infof("Latest block found in the levelDB\r\nheight: %s, hash: %s", header.Number.String(), header.Hash().Hex())
+
+	statediffService, err := createStateDiffService(reader, chainConf, nodeInfo)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -82,7 +94,7 @@ func serve() {
 	if viper.GetBool("prerun.only") {
 		parallel := viper.GetBool("prerun.parallel")
 		if err := statediffService.Run(nil, parallel); err != nil {
-			logWithCommand.Fatal("unable to perform prerun: %v", err)
+			logWithCommand.Fatal("Unable to perform prerun: %v", err)
 		}
 		return
 	}
@@ -112,17 +124,17 @@ func startServers(serv sd.StateDiffService) error {
 	ipcPath := viper.GetString("server.ipcPath")
 	httpPath := viper.GetString("server.httpPath")
 	if ipcPath == "" && httpPath == "" {
-		logWithCommand.Fatal("need an ipc path and/or an http path")
+		logWithCommand.Fatal("Need an ipc path and/or an http path")
 	}
 	if ipcPath != "" {
-		logWithCommand.Info("starting up IPC server")
+		logWithCommand.Info("Starting up IPC server")
 		_, _, err := srpc.StartIPCEndpoint(ipcPath, serv.APIs())
 		if err != nil {
 			return err
 		}
 	}
 	if httpPath != "" {
-		logWithCommand.Info("starting up HTTP server")
+		logWithCommand.Info("Starting up HTTP server")
 		_, err := srpc.StartHTTPEndpoint(httpPath, serv.APIs(), []string{"statediff"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
 		if err != nil {
 			return err

cmd/stats.go

@@ -0,0 +1,54 @@
+// Copyright © 2022 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+// statsCmd represents the serve command
+var statsCmd = &cobra.Command{
+	Use:   "stats",
+	Short: "Report stats for cold levelDB",
+	Long: `Usage
+
+./eth-statediff-service stats --config={path to toml config file}`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *logrus.WithField("SubCommand", subCommand)
+		stats()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(statsCmd)
+}
+
+func stats() {
+	logWithCommand.Info("Running eth-statediff-service stats command")
+
+	reader, _, _ := instantiateLevelDBReader()
+
+	header, err := reader.GetLatestHeader()
+	if err != nil {
+		logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
+	}
+	if header.Number == nil {
+		logWithCommand.Fatal("Latest header found in levelDB has a nil block height")
+	}
+	logWithCommand.Infof("Latest block found in the levelDB\r\nheight: %s, hash: %s", header.Number.String(), header.Hash().Hex())
+}

cmd/util.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/statediff"
 	ind "github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
 	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/spf13/viper"
@@ -17,62 +18,7 @@ import (
 
 type blockRange [2]uint64
 
-func createStateDiffService() (sd.StateDiffService, error) {
+func createStateDiffService(lvlDBReader sd.Reader, chainConf *params.ChainConfig, nodeInfo node.Info) (sd.StateDiffService, error) {
-	// load some necessary params
-	logWithCommand.Info("Loading statediff service parameters")
-	mode := viper.GetString("leveldb.mode")
-	path := viper.GetString("leveldb.path")
-	ancientPath := viper.GetString("leveldb.ancient")
-	url := viper.GetString("leveldb.url")
-
-	if mode == "local" {
-		if path == "" || ancientPath == "" {
-			logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
-		}
-	} else if mode == "remote" {
-		if url == "" {
-			logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
-		}
-	} else {
-		logWithCommand.Fatal("Invalid mode provided for LevelDB access")
-	}
-
-	nodeInfo := getEthNodeInfo()
-
-	var chainConf *params.ChainConfig
-	var err error
-	chainConfigPath := viper.GetString("ethereum.chainConfig")
-
-	if chainConfigPath != "" {
-		chainConf, err = statediff.LoadConfig(chainConfigPath)
-	} else {
-		chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
-	}
-
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-
-	// create LevelDB reader
-	logWithCommand.Info("Creating LevelDB reader")
-	readerConf := sd.LvLDBReaderConfig{
-		TrieConfig: &trie.Config{
-			Cache:     viper.GetInt("cache.trie"),
-			Journal:   "",
-			Preimages: false,
-		},
-		ChainConfig: chainConf,
-		Mode:        mode,
-		Path:        path,
-		AncientPath: ancientPath,
-		Url:         url,
-		DBCacheSize: viper.GetInt("cache.database"),
-	}
-	lvlDBReader, err := sd.NewLvlDBReader(readerConf)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-
 	// create statediff service
 	logWithCommand.Info("Setting up database")
 	conf, err := getConfig(nodeInfo)
@@ -140,3 +86,61 @@ func setupPreRunRanges() []sd.RangeRequest {
 
 	return blockRanges
 }
+
+func instantiateLevelDBReader() (sd.Reader, *params.ChainConfig, node.Info) {
+	// load some necessary params
+	logWithCommand.Info("Loading statediff service parameters")
+	mode := viper.GetString("leveldb.mode")
+	path := viper.GetString("leveldb.path")
+	ancientPath := viper.GetString("leveldb.ancient")
+	url := viper.GetString("leveldb.url")
+
+	if mode == "local" {
+		if path == "" || ancientPath == "" {
+			logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
+		}
+	} else if mode == "remote" {
+		if url == "" {
+			logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
+		}
+	} else {
+		logWithCommand.Fatal("Invalid mode provided for LevelDB access")
+	}
+
+	nodeInfo := getEthNodeInfo()
+
+	var chainConf *params.ChainConfig
+	var err error
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+
+	if chainConfigPath != "" {
+		chainConf, err = statediff.LoadConfig(chainConfigPath)
+	} else {
+		chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
+	}
+
+	if err != nil {
+		logWithCommand.Fatalf("Unable to instantiate chain config: %s", err.Error())
+	}
+
+	// create LevelDB reader
+	logWithCommand.Info("Creating LevelDB reader")
+	readerConf := sd.LvLDBReaderConfig{
+		TrieConfig: &trie.Config{
+			Cache:     viper.GetInt("cache.trie"),
+			Journal:   "",
+			Preimages: false,
+		},
+		ChainConfig: chainConf,
+		Mode:        mode,
+		Path:        path,
+		AncientPath: ancientPath,
+		Url:         url,
+		DBCacheSize: viper.GetInt("cache.database"),
+	}
+	reader, err := sd.NewLvlDBReader(readerConf)
+	if err != nil {
+		logWithCommand.Fatalf("Unable to instantiate levelDB reader: %s", err.Error())
+	}
+	return reader, chainConf, nodeInfo
+}

pkg/reader.go

@@ -16,6 +16,7 @@
 package statediff
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 
@@ -36,6 +37,7 @@ type Reader interface {
 	GetReceiptsByHash(hash common.Hash) (types.Receipts, error)
 	GetTdByHash(hash common.Hash) (*big.Int, error)
 	StateDB() state.Database
+	GetLatestHeader() (*types.Header, error)
 }
 
 // LvlDBReader exposes the necessary Reader methods on lvldb
@@ -129,3 +131,12 @@ func (ldr *LvlDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
 func (ldr *LvlDBReader) StateDB() state.Database {
 	return ldr.stateDB
 }
+
+// GetLatestHeader gets the latest header from the levelDB
+func (ldr *LvlDBReader) GetLatestHeader() (*types.Header, error) {
+	header := rawdb.ReadHeadHeader(ldr.ethDB)
+	if header == nil {
+		return nil, errors.New("unable to read head header")
+	}
+	return header, nil
+}