cerc-io/eth-statediff-service
16
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #110 from cerc-io/ian_util

Browse Source 
latest block height util
This commit is contained in:
Ian Norden 2022-10-10 18:31:14 -05:00 committed by GitHub
commit f6df15cb38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 157 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -297,3 +297,12 @@ An example config file:
```
* NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported statediff data with the data generated in `postgres` mode. Reference: https://www.postgresql.org/docs/14/sql-copy.html
### Stats
The binary includes a `stats` command which reports stats for the offline or remote levelDB.
At this time, the only stat supported is to return the latest/highest block height and hash found the levelDB, this is
useful for determining what the upper limit is for a standalone statediffing process on a given levelDB.
`./eth-statediff-service stats --config={path to toml config file}`

12
cmd/root.go
View File

@ -304,11 +304,11 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
if err != nil {
return nil, err
}
logWithCommand.Infof("configuring service for database type: %s", dbType)
logWithCommand.Infof("Configuring service for database type: %s", dbType)
var indexerConfig interfaces.Config
switch dbType {
case shared.FILE:
logWithCommand.Info("starting in sql file writing mode")
logWithCommand.Info("Starting in sql file writing mode")
fileModeStr := viper.GetString("database.fileMode")
fileMode, err := file.ResolveFileMode(fileModeStr)
@ -318,12 +318,12 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
filePathStr := viper.GetString("database.filePath")
if fileMode == file.SQL && filePathStr == "" {
logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
logWithCommand.Fatal("When operating in sql file writing mode a file path must be provided")
}
fileCsvDirStr := viper.GetString("database.fileCsvDir")
if fileMode == file.CSV && fileCsvDirStr == "" {
logWithCommand.Fatal("when operating in csv file writing mode a directory path must be provided")
logWithCommand.Fatal("When operating in csv file writing mode a directory path must be provided")
}
indexerConfig = file.Config{
@ -332,7 +332,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
FilePath: filePathStr,
}
case shared.DUMP:
logWithCommand.Info("starting in data dump mode")
logWithCommand.Info("Starting in data dump mode")
dumpDstStr := viper.GetString("database.dumpDestination")
dumpDst, err := dump.ResolveDumpType(dumpDstStr)
if err != nil {
@ -349,7 +349,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
}
case shared.POSTGRES:
logWithCommand.Info("starting in postgres mode")
logWithCommand.Info("Starting in postgres mode")
driverTypeStr := viper.GetString("database.driver")
driverType, err := postgres.ResolveDriverType(driverTypeStr)
if err != nil {

22
cmd/serve.go
View File

@ -63,7 +63,19 @@ func serve() {
logWithCommand.Info("Running eth-statediff-service serve command")
logWithCommand.Infof("Parallelism: %d", maxParallelism())
statediffService, err := createStateDiffService()
reader, chainConf, nodeInfo := instantiateLevelDBReader()
// report latest block info
header, err := reader.GetLatestHeader()
if err != nil {
logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
}
if header.Number == nil {
logWithCommand.Fatal("Latest header found in levelDB has a nil block height")
}
logWithCommand.Infof("Latest block found in the levelDB\r\nheight: %s, hash: %s", header.Number.String(), header.Hash().Hex())
statediffService, err := createStateDiffService(reader, chainConf, nodeInfo)
if err != nil {
logWithCommand.Fatal(err)
}
@ -82,7 +94,7 @@ func serve() {
if viper.GetBool("prerun.only") {
parallel := viper.GetBool("prerun.parallel")
if err := statediffService.Run(nil, parallel); err != nil {
logWithCommand.Fatal("unable to perform prerun: %v", err)
logWithCommand.Fatal("Unable to perform prerun: %v", err)
}
return
}
@ -112,17 +124,17 @@ func startServers(serv sd.StateDiffService) error {
ipcPath := viper.GetString("server.ipcPath")
httpPath := viper.GetString("server.httpPath")
if ipcPath == "" && httpPath == "" {
logWithCommand.Fatal("need an ipc path and/or an http path")
logWithCommand.Fatal("Need an ipc path and/or an http path")
}
if ipcPath != "" {
logWithCommand.Info("starting up IPC server")
logWithCommand.Info("Starting up IPC server")
_, _, err := srpc.StartIPCEndpoint(ipcPath, serv.APIs())
if err != nil {
return err
}
}
if httpPath != "" {
logWithCommand.Info("starting up HTTP server")
logWithCommand.Info("Starting up HTTP server")
_, err := srpc.StartHTTPEndpoint(httpPath, serv.APIs(), []string{"statediff"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
if err != nil {
return err

54
cmd/stats.go Normal file
View File

@ -0,0 +1,54 @@
// Copyright © 2022 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
// statsCmd represents the serve command
var statsCmd = &cobra.Command{
Use: "stats",
Short: "Report stats for cold levelDB",
Long: `Usage
./eth-statediff-service stats --config={path to toml config file}`,
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *logrus.WithField("SubCommand", subCommand)
stats()
},
}
func init() {
rootCmd.AddCommand(statsCmd)
}
func stats() {
logWithCommand.Info("Running eth-statediff-service stats command")
reader, _, _ := instantiateLevelDBReader()
header, err := reader.GetLatestHeader()
if err != nil {
logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
}
if header.Number == nil {
logWithCommand.Fatal("Latest header found in levelDB has a nil block height")
}
logWithCommand.Infof("Latest block found in the levelDB\r\nheight: %s, hash: %s", header.Number.String(), header.Hash().Hex())
}

116
cmd/util.go
View File

@ -7,6 +7,7 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff"
ind "github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ethereum/go-ethereum/trie"
"github.com/spf13/viper"
@ -17,62 +18,7 @@ import (
type blockRange [2]uint64
func createStateDiffService() (sd.StateDiffService, error) {
// load some necessary params
logWithCommand.Info("Loading statediff service parameters")
mode := viper.GetString("leveldb.mode")
path := viper.GetString("leveldb.path")
ancientPath := viper.GetString("leveldb.ancient")
url := viper.GetString("leveldb.url")
if mode == "local" {
if path == "" || ancientPath == "" {
logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
}
} else if mode == "remote" {
if url == "" {
logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
}
} else {
logWithCommand.Fatal("Invalid mode provided for LevelDB access")
}
nodeInfo := getEthNodeInfo()
var chainConf *params.ChainConfig
var err error
chainConfigPath := viper.GetString("ethereum.chainConfig")
if chainConfigPath != "" {
chainConf, err = statediff.LoadConfig(chainConfigPath)
} else {
chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
}
if err != nil {
logWithCommand.Fatal(err)
}
// create LevelDB reader
logWithCommand.Info("Creating LevelDB reader")
readerConf := sd.LvLDBReaderConfig{
TrieConfig: &trie.Config{
Cache: viper.GetInt("cache.trie"),
Journal: "",
Preimages: false,
},
ChainConfig: chainConf,
Mode: mode,
Path: path,
AncientPath: ancientPath,
Url: url,
DBCacheSize: viper.GetInt("cache.database"),
}
lvlDBReader, err := sd.NewLvlDBReader(readerConf)
if err != nil {
logWithCommand.Fatal(err)
}
func createStateDiffService(lvlDBReader sd.Reader, chainConf *params.ChainConfig, nodeInfo node.Info) (sd.StateDiffService, error) {
// create statediff service
logWithCommand.Info("Setting up database")
conf, err := getConfig(nodeInfo)
@ -140,3 +86,61 @@ func setupPreRunRanges() []sd.RangeRequest {
return blockRanges
}
func instantiateLevelDBReader() (sd.Reader, *params.ChainConfig, node.Info) {
// load some necessary params
logWithCommand.Info("Loading statediff service parameters")
mode := viper.GetString("leveldb.mode")
path := viper.GetString("leveldb.path")
ancientPath := viper.GetString("leveldb.ancient")
url := viper.GetString("leveldb.url")
if mode == "local" {
if path == "" || ancientPath == "" {
logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
}
} else if mode == "remote" {
if url == "" {
logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
}
} else {
logWithCommand.Fatal("Invalid mode provided for LevelDB access")
}
nodeInfo := getEthNodeInfo()
var chainConf *params.ChainConfig
var err error
chainConfigPath := viper.GetString("ethereum.chainConfig")
if chainConfigPath != "" {
chainConf, err = statediff.LoadConfig(chainConfigPath)
} else {
chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
}
if err != nil {
logWithCommand.Fatalf("Unable to instantiate chain config: %s", err.Error())
}
// create LevelDB reader
logWithCommand.Info("Creating LevelDB reader")
readerConf := sd.LvLDBReaderConfig{
TrieConfig: &trie.Config{
Cache: viper.GetInt("cache.trie"),
Journal: "",
Preimages: false,
},
ChainConfig: chainConf,
Mode: mode,
Path: path,
AncientPath: ancientPath,
Url: url,
DBCacheSize: viper.GetInt("cache.database"),
}
reader, err := sd.NewLvlDBReader(readerConf)
if err != nil {
logWithCommand.Fatalf("Unable to instantiate levelDB reader: %s", err.Error())
}
return reader, chainConf, nodeInfo
}

11
pkg/reader.go
View File

@ -16,6 +16,7 @@
package statediff
import (
"errors"
"fmt"
"math/big"
@ -36,6 +37,7 @@ type Reader interface {
GetReceiptsByHash(hash common.Hash) (types.Receipts, error)
GetTdByHash(hash common.Hash) (*big.Int, error)
StateDB() state.Database
GetLatestHeader() (*types.Header, error)
}
// LvlDBReader exposes the necessary Reader methods on lvldb
@ -129,3 +131,12 @@ func (ldr *LvlDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
func (ldr *LvlDBReader) StateDB() state.Database {
return ldr.stateDB
}
// GetLatestHeader gets the latest header from the levelDB
func (ldr *LvlDBReader) GetLatestHeader() (*types.Header, error) {
header := rawdb.ReadHeadHeader(ldr.ethDB)
if header == nil {
return nil, errors.New("unable to read head header")
}
return header, nil
}