2022-05-24 18:39:40 +00:00
// Copyright 2021 The go-ethereum Authors
2021-02-23 10:27:32 +00:00
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
2021-11-02 10:31:45 +00:00
"bytes"
2021-02-23 10:27:32 +00:00
"fmt"
"os"
2021-11-02 10:31:45 +00:00
"os/signal"
2021-02-23 10:27:32 +00:00
"path/filepath"
2021-03-30 11:57:21 +00:00
"strconv"
2021-11-02 10:31:45 +00:00
"strings"
"syscall"
2021-02-23 10:27:32 +00:00
"time"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/console/prompt"
"github.com/ethereum/go-ethereum/core/rawdb"
2022-01-18 10:30:41 +00:00
"github.com/ethereum/go-ethereum/core/state/snapshot"
2022-05-17 11:01:46 +00:00
"github.com/ethereum/go-ethereum/crypto"
2021-02-23 10:27:32 +00:00
"github.com/ethereum/go-ethereum/ethdb"
2022-07-25 09:07:44 +00:00
"github.com/ethereum/go-ethereum/internal/flags"
2021-02-23 10:27:32 +00:00
"github.com/ethereum/go-ethereum/log"
2021-03-30 11:57:21 +00:00
"github.com/ethereum/go-ethereum/trie"
2022-01-18 10:30:41 +00:00
"github.com/olekukonko/tablewriter"
2022-06-27 16:22:36 +00:00
"github.com/urfave/cli/v2"
2021-02-23 10:27:32 +00:00
)
var (
2022-06-27 16:22:36 +00:00
removedbCommand = & cli . Command {
Action : removeDB ,
2021-02-23 10:27:32 +00:00
Name : "removedb" ,
Usage : "Remove blockchain and state databases" ,
ArgsUsage : "" ,
2023-09-22 11:56:48 +00:00
Flags : utils . DatabaseFlags ,
2021-02-23 10:27:32 +00:00
Description : `
Remove blockchain and state databases ` ,
}
2022-06-27 16:22:36 +00:00
dbCommand = & cli . Command {
2021-02-23 10:27:32 +00:00
Name : "db" ,
Usage : "Low level database operations" ,
ArgsUsage : "" ,
2022-06-27 16:22:36 +00:00
Subcommands : [ ] * cli . Command {
2021-02-23 10:27:32 +00:00
dbInspectCmd ,
dbStatCmd ,
dbCompactCmd ,
dbGetCmd ,
dbDeleteCmd ,
dbPutCmd ,
2021-03-30 11:57:21 +00:00
dbGetSlotsCmd ,
2021-04-13 13:45:30 +00:00
dbDumpFreezerIndex ,
2021-11-02 10:31:45 +00:00
dbImportCmd ,
dbExportCmd ,
2022-01-18 10:30:41 +00:00
dbMetadataCmd ,
2022-05-17 11:01:46 +00:00
dbCheckStateContentCmd ,
2021-02-23 10:27:32 +00:00
} ,
}
2022-06-27 16:22:36 +00:00
dbInspectCmd = & cli . Command {
Action : inspect ,
2021-02-23 10:27:32 +00:00
Name : "inspect" ,
ArgsUsage : "<prefix> <start>" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-03-22 18:06:30 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2021-02-23 10:27:32 +00:00
Usage : "Inspect the storage size for each type of data in the database" ,
Description : ` This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data. ` ,
}
2022-06-27 16:22:36 +00:00
dbCheckStateContentCmd = & cli . Command {
Action : checkStateContent ,
2022-05-17 11:01:46 +00:00
Name : "check-state-content" ,
ArgsUsage : "<start (optional)>" ,
2023-09-22 11:56:48 +00:00
Flags : flags . Merge ( utils . NetworkFlags , utils . DatabaseFlags ) ,
2022-05-17 11:01:46 +00:00
Usage : "Verify that state data is cryptographically correct" ,
Description : ` This command iterates the entire database for 32 - byte keys , looking for rlp - encoded trie nodes .
For each trie node encountered , it checks that the key corresponds to the keccak256 ( value ) . If this is not true , this indicates
a data corruption . ` ,
}
2022-06-27 16:22:36 +00:00
dbStatCmd = & cli . Command {
Action : dbStats ,
2021-02-23 10:27:32 +00:00
Name : "stats" ,
Usage : "Print leveldb statistics" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-03-22 18:06:30 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2021-02-23 10:27:32 +00:00
}
2022-06-27 16:22:36 +00:00
dbCompactCmd = & cli . Command {
Action : dbCompact ,
2021-02-23 10:27:32 +00:00
Name : "compact" ,
Usage : "Compact leveldb database. WARNING: May take a very long time" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-03-22 18:06:30 +00:00
utils . SyncModeFlag ,
utils . CacheFlag ,
utils . CacheDatabaseFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2023-11-10 12:26:13 +00:00
Description : ` This command performs a database compaction .
2021-02-23 10:27:32 +00:00
WARNING : This operation may take a very long time to finish , and may cause database
corruption if it is aborted during execution ' ! ` ,
}
2022-06-27 16:22:36 +00:00
dbGetCmd = & cli . Command {
Action : dbGet ,
2021-03-22 18:06:30 +00:00
Name : "get" ,
Usage : "Show the value of a database key" ,
ArgsUsage : "<hex-encoded key>" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-03-22 18:06:30 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2021-02-23 10:27:32 +00:00
Description : "This command looks up the specified database key from the database." ,
}
2022-06-27 16:22:36 +00:00
dbDeleteCmd = & cli . Command {
Action : dbDelete ,
2021-02-23 10:27:32 +00:00
Name : "delete" ,
Usage : "Delete a database key (WARNING: may corrupt your database)" ,
ArgsUsage : "<hex-encoded key>" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-03-22 18:06:30 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2023-11-10 12:26:13 +00:00
Description : ` This command deletes the specified database key from the database .
2021-02-23 10:27:32 +00:00
WARNING : This is a low - level operation which may cause database corruption ! ` ,
}
2022-06-27 16:22:36 +00:00
dbPutCmd = & cli . Command {
Action : dbPut ,
2021-02-23 10:27:32 +00:00
Name : "put" ,
Usage : "Set the value of a database key (WARNING: may corrupt your database)" ,
ArgsUsage : "<hex-encoded key> <hex-encoded value>" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-03-22 18:06:30 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2023-11-10 12:26:13 +00:00
Description : ` This command sets a given database key to the given value .
2021-02-23 10:27:32 +00:00
WARNING : This is a low - level operation which may cause database corruption ! ` ,
}
2022-06-27 16:22:36 +00:00
dbGetSlotsCmd = & cli . Command {
Action : dbDumpTrie ,
2021-03-30 11:57:21 +00:00
Name : "dumptrie" ,
Usage : "Show the storage key/values of a given storage trie" ,
cmd, core, eth, les, light: track deleted nodes (#25757)
* cmd, core, eth, les, light: track deleted nodes
* trie: add docs
* trie: address comments
* cmd, core, eth, les, light, trie: trie id
* trie: add tests
* trie, core: updates
* trie: fix imports
* trie: add utility print-method for nodeset
* trie: import err
* trie: fix go vet warnings
Co-authored-by: Martin Holst Swende <martin@swende.se>
2022-09-27 08:01:02 +00:00
ArgsUsage : "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-03-30 11:57:21 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2021-03-30 11:57:21 +00:00
Description : "This command looks up the specified database key from the database." ,
}
2022-06-27 16:22:36 +00:00
dbDumpFreezerIndex = & cli . Command {
Action : freezerInspect ,
2021-04-13 13:45:30 +00:00
Name : "freezer-index" ,
2022-08-08 09:08:36 +00:00
Usage : "Dump out the index of a specific freezer table" ,
ArgsUsage : "<freezer-type> <table-type> <start (int)> <end (int)>" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-04-13 13:45:30 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2021-04-13 13:45:30 +00:00
Description : "This command displays information about the freezer index." ,
}
2022-06-27 16:22:36 +00:00
dbImportCmd = & cli . Command {
Action : importLDBdata ,
2021-11-02 10:31:45 +00:00
Name : "import" ,
Usage : "Imports leveldb-data from an exported RLP dump." ,
ArgsUsage : "<dumpfile> <start (optional)" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-11-02 10:31:45 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2021-11-02 10:31:45 +00:00
Description : "The import command imports the specific chain data from an RLP encoded stream." ,
}
2022-06-27 16:22:36 +00:00
dbExportCmd = & cli . Command {
Action : exportChaindata ,
2021-11-02 10:31:45 +00:00
Name : "export" ,
Usage : "Exports the chain data into an RLP dump. If the <dumpfile> has .gz suffix, gzip compression will be used." ,
ArgsUsage : "<type> <dumpfile>" ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2021-11-02 10:31:45 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2021-11-02 10:31:45 +00:00
Description : "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed." ,
}
2022-06-27 16:22:36 +00:00
dbMetadataCmd = & cli . Command {
Action : showMetaData ,
2022-01-18 10:30:41 +00:00
Name : "metadata" ,
Usage : "Shows metadata about the chain status." ,
2022-07-25 09:07:44 +00:00
Flags : flags . Merge ( [ ] cli . Flag {
2022-01-18 10:30:41 +00:00
utils . SyncModeFlag ,
2023-09-22 11:56:48 +00:00
} , utils . NetworkFlags , utils . DatabaseFlags ) ,
2022-01-18 10:30:41 +00:00
Description : "Shows metadata about the chain status." ,
}
2021-02-23 10:27:32 +00:00
)
func removeDB ( ctx * cli . Context ) error {
stack , config := makeConfigNode ( ctx )
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-21 19:28:32 +00:00
// Resolve folder paths.
var (
rootDir = stack . ResolvePath ( "chaindata" )
ancientDir = config . Eth . DatabaseFreezer
)
2021-02-23 10:27:32 +00:00
switch {
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-21 19:28:32 +00:00
case ancientDir == "" :
ancientDir = filepath . Join ( stack . ResolvePath ( "chaindata" ) , "ancient" )
case ! filepath . IsAbs ( ancientDir ) :
ancientDir = config . Node . ResolvePath ( ancientDir )
2021-02-23 10:27:32 +00:00
}
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-21 19:28:32 +00:00
// Delete state data
statePaths := [ ] string { rootDir , filepath . Join ( ancientDir , rawdb . StateFreezerName ) }
confirmAndRemoveDB ( statePaths , "state data" )
// Delete ancient chain
chainPaths := [ ] string { filepath . Join ( ancientDir , rawdb . ChainFreezerName ) }
confirmAndRemoveDB ( chainPaths , "ancient chain" )
2021-02-23 10:27:32 +00:00
return nil
}
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-21 19:28:32 +00:00
// removeFolder deletes all files (not folders) inside the directory 'dir' (but
// not files in subfolders).
func removeFolder ( dir string ) {
filepath . Walk ( dir , func ( path string , info os . FileInfo , err error ) error {
// If we're at the top level folder, recurse into
if path == dir {
return nil
}
// Delete all the files, but not subfolders
if ! info . IsDir ( ) {
os . Remove ( path )
return nil
}
return filepath . SkipDir
} )
}
2021-02-23 10:27:32 +00:00
// confirmAndRemoveDB prompts the user for a last confirmation and removes the
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-21 19:28:32 +00:00
// list of folders if accepted.
func confirmAndRemoveDB ( paths [ ] string , kind string ) {
msg := fmt . Sprintf ( "Location(s) of '%s': \n" , kind )
for _ , path := range paths {
msg += fmt . Sprintf ( "\t- %s\n" , path )
}
fmt . Println ( msg )
confirm , err := prompt . Stdin . PromptConfirm ( fmt . Sprintf ( "Remove '%s'?" , kind ) )
2021-02-23 10:27:32 +00:00
switch {
case err != nil :
utils . Fatalf ( "%v" , err )
case ! confirm :
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-21 19:28:32 +00:00
log . Info ( "Database deletion skipped" , "kind" , kind , "paths" , paths )
2021-02-23 10:27:32 +00:00
default :
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-21 19:28:32 +00:00
var (
deleted [ ] string
start = time . Now ( )
)
for _ , path := range paths {
if common . FileExist ( path ) {
removeFolder ( path )
deleted = append ( deleted , path )
} else {
log . Info ( "Folder is not existent" , "path" , path )
2021-02-23 10:27:32 +00:00
}
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-21 19:28:32 +00:00
}
log . Info ( "Database successfully deleted" , "kind" , kind , "paths" , deleted , "elapsed" , common . PrettyDuration ( time . Since ( start ) ) )
2021-02-23 10:27:32 +00:00
}
}
func inspect ( ctx * cli . Context ) error {
var (
prefix [ ] byte
start [ ] byte
)
if ctx . NArg ( ) > 2 {
2022-08-08 09:08:36 +00:00
return fmt . Errorf ( "max 2 arguments: %v" , ctx . Command . ArgsUsage )
2021-02-23 10:27:32 +00:00
}
if ctx . NArg ( ) >= 1 {
if d , err := hexutil . Decode ( ctx . Args ( ) . Get ( 0 ) ) ; err != nil {
return fmt . Errorf ( "failed to hex-decode 'prefix': %v" , err )
} else {
prefix = d
}
}
if ctx . NArg ( ) >= 2 {
if d , err := hexutil . Decode ( ctx . Args ( ) . Get ( 1 ) ) ; err != nil {
return fmt . Errorf ( "failed to hex-decode 'start': %v" , err )
} else {
start = d
}
}
stack , _ := makeConfigNode ( ctx )
defer stack . Close ( )
2021-03-22 18:06:30 +00:00
db := utils . MakeChainDatabase ( ctx , stack , true )
defer db . Close ( )
2021-02-23 10:27:32 +00:00
2021-03-22 18:06:30 +00:00
return rawdb . InspectDatabase ( db , prefix , start )
2021-02-23 10:27:32 +00:00
}
2022-05-17 11:01:46 +00:00
func checkStateContent ( ctx * cli . Context ) error {
var (
prefix [ ] byte
start [ ] byte
)
if ctx . NArg ( ) > 1 {
2022-05-30 10:37:42 +00:00
return fmt . Errorf ( "max 1 argument: %v" , ctx . Command . ArgsUsage )
2022-05-17 11:01:46 +00:00
}
if ctx . NArg ( ) > 0 {
if d , err := hexutil . Decode ( ctx . Args ( ) . First ( ) ) ; err != nil {
return fmt . Errorf ( "failed to hex-decode 'start': %v" , err )
} else {
start = d
}
}
stack , _ := makeConfigNode ( ctx )
defer stack . Close ( )
db := utils . MakeChainDatabase ( ctx , stack , true )
defer db . Close ( )
var (
it = rawdb . NewKeyLengthIterator ( db . NewIterator ( prefix , start ) , 32 )
hasher = crypto . NewKeccakState ( )
got = make ( [ ] byte , 32 )
errs int
count int
startTime = time . Now ( )
lastLog = time . Now ( )
)
for it . Next ( ) {
count ++
k := it . Key ( )
2022-05-30 10:37:42 +00:00
v := it . Value ( )
2022-05-17 11:01:46 +00:00
hasher . Reset ( )
hasher . Write ( v )
hasher . Read ( got )
if ! bytes . Equal ( k , got ) {
errs ++
2022-07-04 08:03:32 +00:00
fmt . Printf ( "Error at %#x\n" , k )
fmt . Printf ( " Hash: %#x\n" , got )
fmt . Printf ( " Data: %#x\n" , v )
2022-05-17 11:01:46 +00:00
}
if time . Since ( lastLog ) > 8 * time . Second {
log . Info ( "Iterating the database" , "at" , fmt . Sprintf ( "%#x" , k ) , "elapsed" , common . PrettyDuration ( time . Since ( startTime ) ) )
lastLog = time . Now ( )
}
}
if err := it . Error ( ) ; err != nil {
return err
}
log . Info ( "Iterated the state content" , "errors" , errs , "items" , count )
return nil
}
2022-05-06 11:28:42 +00:00
func showLeveldbStats ( db ethdb . KeyValueStater ) {
2021-02-23 10:27:32 +00:00
if stats , err := db . Stat ( "leveldb.stats" ) ; err != nil {
log . Warn ( "Failed to read database stats" , "error" , err )
} else {
fmt . Println ( stats )
}
if ioStats , err := db . Stat ( "leveldb.iostats" ) ; err != nil {
log . Warn ( "Failed to read database iostats" , "error" , err )
} else {
fmt . Println ( ioStats )
}
}
func dbStats ( ctx * cli . Context ) error {
stack , _ := makeConfigNode ( ctx )
defer stack . Close ( )
2021-03-22 18:06:30 +00:00
db := utils . MakeChainDatabase ( ctx , stack , true )
defer db . Close ( )
2021-02-23 10:27:32 +00:00
showLeveldbStats ( db )
return nil
}
func dbCompact ( ctx * cli . Context ) error {
stack , _ := makeConfigNode ( ctx )
defer stack . Close ( )
2021-03-22 18:06:30 +00:00
db := utils . MakeChainDatabase ( ctx , stack , false )
defer db . Close ( )
log . Info ( "Stats before compaction" )
2021-02-23 10:27:32 +00:00
showLeveldbStats ( db )
2021-03-22 18:06:30 +00:00
2021-02-23 10:27:32 +00:00
log . Info ( "Triggering compaction" )
2021-03-22 18:06:30 +00:00
if err := db . Compact ( nil , nil ) ; err != nil {
2021-02-23 10:27:32 +00:00
log . Info ( "Compact err" , "error" , err )
2021-03-22 18:06:30 +00:00
return err
2021-02-23 10:27:32 +00:00
}
2021-03-22 18:06:30 +00:00
log . Info ( "Stats after compaction" )
2021-02-23 10:27:32 +00:00
showLeveldbStats ( db )
2021-03-22 18:06:30 +00:00
return nil
2021-02-23 10:27:32 +00:00
}
// dbGet shows the value of a given database key
func dbGet ( ctx * cli . Context ) error {
if ctx . NArg ( ) != 1 {
return fmt . Errorf ( "required arguments: %v" , ctx . Command . ArgsUsage )
}
stack , _ := makeConfigNode ( ctx )
defer stack . Close ( )
2021-03-22 18:06:30 +00:00
db := utils . MakeChainDatabase ( ctx , stack , true )
2021-02-23 10:27:32 +00:00
defer db . Close ( )
2021-03-22 18:06:30 +00:00
2022-04-27 06:37:48 +00:00
key , err := common . ParseHexOrString ( ctx . Args ( ) . Get ( 0 ) )
2021-02-23 10:27:32 +00:00
if err != nil {
log . Info ( "Could not decode the key" , "error" , err )
return err
}
2021-10-18 10:18:49 +00:00
2021-02-23 10:27:32 +00:00
data , err := db . Get ( key )
if err != nil {
2022-07-18 11:22:56 +00:00
log . Info ( "Get operation failed" , "key" , fmt . Sprintf ( "%#x" , key ) , "error" , err )
2021-02-23 10:27:32 +00:00
return err
}
2021-03-22 18:06:30 +00:00
fmt . Printf ( "key %#x: %#x\n" , key , data )
2021-02-23 10:27:32 +00:00
return nil
}
// dbDelete deletes a key from the database
func dbDelete ( ctx * cli . Context ) error {
if ctx . NArg ( ) != 1 {
return fmt . Errorf ( "required arguments: %v" , ctx . Command . ArgsUsage )
}
stack , _ := makeConfigNode ( ctx )
defer stack . Close ( )
2021-03-22 18:06:30 +00:00
db := utils . MakeChainDatabase ( ctx , stack , false )
2021-02-23 10:27:32 +00:00
defer db . Close ( )
2021-03-22 18:06:30 +00:00
2022-04-27 06:37:48 +00:00
key , err := common . ParseHexOrString ( ctx . Args ( ) . Get ( 0 ) )
2021-02-23 10:27:32 +00:00
if err != nil {
log . Info ( "Could not decode the key" , "error" , err )
return err
}
2021-03-22 18:06:30 +00:00
data , err := db . Get ( key )
if err == nil {
fmt . Printf ( "Previous value: %#x\n" , data )
}
2021-02-23 10:27:32 +00:00
if err = db . Delete ( key ) ; err != nil {
2022-07-18 11:22:56 +00:00
log . Info ( "Delete operation returned an error" , "key" , fmt . Sprintf ( "%#x" , key ) , "error" , err )
2021-02-23 10:27:32 +00:00
return err
}
return nil
}
// dbPut overwrite a value in the database
func dbPut ( ctx * cli . Context ) error {
if ctx . NArg ( ) != 2 {
return fmt . Errorf ( "required arguments: %v" , ctx . Command . ArgsUsage )
}
stack , _ := makeConfigNode ( ctx )
defer stack . Close ( )
2021-03-22 18:06:30 +00:00
db := utils . MakeChainDatabase ( ctx , stack , false )
2021-02-23 10:27:32 +00:00
defer db . Close ( )
2021-03-22 18:06:30 +00:00
2021-02-23 10:27:32 +00:00
var (
key [ ] byte
value [ ] byte
data [ ] byte
err error
)
2022-04-27 06:37:48 +00:00
key , err = common . ParseHexOrString ( ctx . Args ( ) . Get ( 0 ) )
2021-02-23 10:27:32 +00:00
if err != nil {
log . Info ( "Could not decode the key" , "error" , err )
return err
}
value , err = hexutil . Decode ( ctx . Args ( ) . Get ( 1 ) )
if err != nil {
log . Info ( "Could not decode the value" , "error" , err )
return err
}
data , err = db . Get ( key )
if err == nil {
2021-03-22 18:06:30 +00:00
fmt . Printf ( "Previous value: %#x\n" , data )
2021-02-23 10:27:32 +00:00
}
return db . Put ( key , value )
}
2021-03-30 11:57:21 +00:00
// dbDumpTrie shows the key-value slots of a given storage trie
func dbDumpTrie ( ctx * cli . Context ) error {
cmd, core, eth, les, light: track deleted nodes (#25757)
* cmd, core, eth, les, light: track deleted nodes
* trie: add docs
* trie: address comments
* cmd, core, eth, les, light, trie: trie id
* trie: add tests
* trie, core: updates
* trie: fix imports
* trie: add utility print-method for nodeset
* trie: import err
* trie: fix go vet warnings
Co-authored-by: Martin Holst Swende <martin@swende.se>
2022-09-27 08:01:02 +00:00
if ctx . NArg ( ) < 3 {
2021-03-30 11:57:21 +00:00
return fmt . Errorf ( "required arguments: %v" , ctx . Command . ArgsUsage )
}
stack , _ := makeConfigNode ( ctx )
defer stack . Close ( )
db := utils . MakeChainDatabase ( ctx , stack , true )
defer db . Close ( )
cmd, core, eth, les, light: track deleted nodes (#25757)
* cmd, core, eth, les, light: track deleted nodes
* trie: add docs
* trie: address comments
* cmd, core, eth, les, light, trie: trie id
* trie: add tests
* trie, core: updates
* trie: fix imports
* trie: add utility print-method for nodeset
* trie: import err
* trie: fix go vet warnings
Co-authored-by: Martin Holst Swende <martin@swende.se>
2022-09-27 08:01:02 +00:00
2023-11-14 12:09:40 +00:00
triedb := utils . MakeTrieDatabase ( ctx , db , false , true , false )
all: activate pbss as experimental feature (#26274)
* all: activate pbss
* core/rawdb: fix compilation error
* cma, core, eth, les, trie: address comments
* cmd, core, eth, trie: polish code
* core, cmd, eth: address comments
* cmd, core, eth, les, light, tests: address comment
* cmd/utils: shorten log message
* trie/triedb/pathdb: limit node buffer size to 1gb
* cmd/utils: fix opening non-existing db
* cmd/utils: rename flag name
* cmd, core: group chain history flags and fix tests
* core, eth, trie: fix memory leak in snapshot generation
* cmd, eth, internal: deprecate flags
* all: enable state tests for pathdb, fixes
* cmd, core: polish code
* trie/triedb/pathdb: limit the node buffer size to 256mb
---------
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-08-10 19:21:36 +00:00
defer triedb . Close ( )
2021-03-30 11:57:21 +00:00
var (
cmd, core, eth, les, light: track deleted nodes (#25757)
* cmd, core, eth, les, light: track deleted nodes
* trie: add docs
* trie: address comments
* cmd, core, eth, les, light, trie: trie id
* trie: add tests
* trie, core: updates
* trie: fix imports
* trie: add utility print-method for nodeset
* trie: import err
* trie: fix go vet warnings
Co-authored-by: Martin Holst Swende <martin@swende.se>
2022-09-27 08:01:02 +00:00
state [ ] byte
storage [ ] byte
account [ ] byte
start [ ] byte
max = int64 ( - 1 )
err error
2021-03-30 11:57:21 +00:00
)
cmd, core, eth, les, light: track deleted nodes (#25757)
* cmd, core, eth, les, light: track deleted nodes
* trie: add docs
* trie: address comments
* cmd, core, eth, les, light, trie: trie id
* trie: add tests
* trie, core: updates
* trie: fix imports
* trie: add utility print-method for nodeset
* trie: import err
* trie: fix go vet warnings
Co-authored-by: Martin Holst Swende <martin@swende.se>
2022-09-27 08:01:02 +00:00
if state , err = hexutil . Decode ( ctx . Args ( ) . Get ( 0 ) ) ; err != nil {
log . Info ( "Could not decode the state root" , "error" , err )
2021-03-30 11:57:21 +00:00
return err
}
cmd, core, eth, les, light: track deleted nodes (#25757)
* cmd, core, eth, les, light: track deleted nodes
* trie: add docs
* trie: address comments
* cmd, core, eth, les, light, trie: trie id
* trie: add tests
* trie, core: updates
* trie: fix imports
* trie: add utility print-method for nodeset
* trie: import err
* trie: fix go vet warnings
Co-authored-by: Martin Holst Swende <martin@swende.se>
2022-09-27 08:01:02 +00:00
if account , err = hexutil . Decode ( ctx . Args ( ) . Get ( 1 ) ) ; err != nil {
log . Info ( "Could not decode the account hash" , "error" , err )
return err
}
if storage , err = hexutil . Decode ( ctx . Args ( ) . Get ( 2 ) ) ; err != nil {
log . Info ( "Could not decode the storage trie root" , "error" , err )
return err
}
if ctx . NArg ( ) > 3 {
if start , err = hexutil . Decode ( ctx . Args ( ) . Get ( 3 ) ) ; err != nil {
2021-03-30 11:57:21 +00:00
log . Info ( "Could not decode the seek position" , "error" , err )
return err
}
}
cmd, core, eth, les, light: track deleted nodes (#25757)
* cmd, core, eth, les, light: track deleted nodes
* trie: add docs
* trie: address comments
* cmd, core, eth, les, light, trie: trie id
* trie: add tests
* trie, core: updates
* trie: fix imports
* trie: add utility print-method for nodeset
* trie: import err
* trie: fix go vet warnings
Co-authored-by: Martin Holst Swende <martin@swende.se>
2022-09-27 08:01:02 +00:00
if ctx . NArg ( ) > 4 {
if max , err = strconv . ParseInt ( ctx . Args ( ) . Get ( 4 ) , 10 , 64 ) ; err != nil {
2021-03-30 11:57:21 +00:00
log . Info ( "Could not decode the max count" , "error" , err )
return err
}
}
cmd, core, eth, les, light: track deleted nodes (#25757)
* cmd, core, eth, les, light: track deleted nodes
* trie: add docs
* trie: address comments
* cmd, core, eth, les, light, trie: trie id
* trie: add tests
* trie, core: updates
* trie: fix imports
* trie: add utility print-method for nodeset
* trie: import err
* trie: fix go vet warnings
Co-authored-by: Martin Holst Swende <martin@swende.se>
2022-09-27 08:01:02 +00:00
id := trie . StorageTrieID ( common . BytesToHash ( state ) , common . BytesToHash ( account ) , common . BytesToHash ( storage ) )
all: activate pbss as experimental feature (#26274)
* all: activate pbss
* core/rawdb: fix compilation error
* cma, core, eth, les, trie: address comments
* cmd, core, eth, trie: polish code
* core, cmd, eth: address comments
* cmd, core, eth, les, light, tests: address comment
* cmd/utils: shorten log message
* trie/triedb/pathdb: limit node buffer size to 1gb
* cmd/utils: fix opening non-existing db
* cmd/utils: rename flag name
* cmd, core: group chain history flags and fix tests
* core, eth, trie: fix memory leak in snapshot generation
* cmd, eth, internal: deprecate flags
* all: enable state tests for pathdb, fixes
* cmd, core: polish code
* trie/triedb/pathdb: limit the node buffer size to 256mb
---------
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-08-10 19:21:36 +00:00
theTrie , err := trie . New ( id , triedb )
2021-03-30 11:57:21 +00:00
if err != nil {
return err
}
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 19:31:45 +00:00
trieIt , err := theTrie . NodeIterator ( start )
if err != nil {
return err
}
2021-03-30 11:57:21 +00:00
var count int64
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 19:31:45 +00:00
it := trie . NewIterator ( trieIt )
2021-03-30 11:57:21 +00:00
for it . Next ( ) {
if max > 0 && count == max {
fmt . Printf ( "Exiting after %d values\n" , count )
break
}
fmt . Printf ( " %d. key %#x: %#x\n" , count , it . Key , it . Value )
count ++
}
return it . Err
}
2021-04-13 13:45:30 +00:00
func freezerInspect ( ctx * cli . Context ) error {
2022-08-08 09:08:36 +00:00
if ctx . NArg ( ) < 4 {
2021-04-13 13:45:30 +00:00
return fmt . Errorf ( "required arguments: %v" , ctx . Command . ArgsUsage )
}
2022-08-08 09:08:36 +00:00
var (
freezer = ctx . Args ( ) . Get ( 0 )
table = ctx . Args ( ) . Get ( 1 )
)
start , err := strconv . ParseInt ( ctx . Args ( ) . Get ( 2 ) , 10 , 64 )
if err != nil {
log . Info ( "Could not read start-param" , "err" , err )
2021-04-13 13:45:30 +00:00
return err
}
2022-08-08 09:08:36 +00:00
end , err := strconv . ParseInt ( ctx . Args ( ) . Get ( 3 ) , 10 , 64 )
if err != nil {
log . Info ( "Could not read count param" , "err" , err )
2021-04-13 13:45:30 +00:00
return err
}
stack , _ := makeConfigNode ( ctx )
2023-01-16 08:57:27 +00:00
ancient := stack . ResolveAncient ( "chaindata" , ctx . String ( utils . AncientFlag . Name ) )
stack . Close ( )
2022-08-08 09:08:36 +00:00
return rawdb . InspectFreezerTable ( ancient , freezer , table , start , end )
2021-04-13 13:45:30 +00:00
}
2021-10-18 10:18:49 +00:00
2021-11-02 10:31:45 +00:00
func importLDBdata ( ctx * cli . Context ) error {
start := 0
switch ctx . NArg ( ) {
case 1 :
break
case 2 :
s , err := strconv . Atoi ( ctx . Args ( ) . Get ( 1 ) )
if err != nil {
return fmt . Errorf ( "second arg must be an integer: %v" , err )
}
start = s
default :
return fmt . Errorf ( "required arguments: %v" , ctx . Command . ArgsUsage )
}
var (
fName = ctx . Args ( ) . Get ( 0 )
stack , _ = makeConfigNode ( ctx )
interrupt = make ( chan os . Signal , 1 )
stop = make ( chan struct { } )
)
defer stack . Close ( )
signal . Notify ( interrupt , syscall . SIGINT , syscall . SIGTERM )
defer signal . Stop ( interrupt )
defer close ( interrupt )
go func ( ) {
if _ , ok := <- interrupt ; ok {
log . Info ( "Interrupted during ldb import, stopping at next batch" )
}
close ( stop )
} ( )
db := utils . MakeChainDatabase ( ctx , stack , false )
2023-09-25 16:28:20 +00:00
defer db . Close ( )
2021-11-02 10:31:45 +00:00
return utils . ImportLDBData ( db , fName , int64 ( start ) , stop )
}
type preimageIterator struct {
iter ethdb . Iterator
}
func ( iter * preimageIterator ) Next ( ) ( byte , [ ] byte , [ ] byte , bool ) {
for iter . iter . Next ( ) {
key := iter . iter . Key ( )
if bytes . HasPrefix ( key , rawdb . PreimagePrefix ) && len ( key ) == ( len ( rawdb . PreimagePrefix ) + common . HashLength ) {
return utils . OpBatchAdd , key , iter . iter . Value ( ) , true
}
}
return 0 , nil , nil , false
}
func ( iter * preimageIterator ) Release ( ) {
iter . iter . Release ( )
}
type snapshotIterator struct {
init bool
account ethdb . Iterator
storage ethdb . Iterator
}
func ( iter * snapshotIterator ) Next ( ) ( byte , [ ] byte , [ ] byte , bool ) {
if ! iter . init {
iter . init = true
return utils . OpBatchDel , rawdb . SnapshotRootKey , nil , true
}
for iter . account . Next ( ) {
key := iter . account . Key ( )
if bytes . HasPrefix ( key , rawdb . SnapshotAccountPrefix ) && len ( key ) == ( len ( rawdb . SnapshotAccountPrefix ) + common . HashLength ) {
return utils . OpBatchAdd , key , iter . account . Value ( ) , true
}
}
for iter . storage . Next ( ) {
key := iter . storage . Key ( )
if bytes . HasPrefix ( key , rawdb . SnapshotStoragePrefix ) && len ( key ) == ( len ( rawdb . SnapshotStoragePrefix ) + 2 * common . HashLength ) {
return utils . OpBatchAdd , key , iter . storage . Value ( ) , true
}
}
return 0 , nil , nil , false
}
func ( iter * snapshotIterator ) Release ( ) {
iter . account . Release ( )
iter . storage . Release ( )
}
// chainExporters defines the export scheme for all exportable chain data.
var chainExporters = map [ string ] func ( db ethdb . Database ) utils . ChainDataIterator {
"preimage" : func ( db ethdb . Database ) utils . ChainDataIterator {
iter := db . NewIterator ( rawdb . PreimagePrefix , nil )
return & preimageIterator { iter : iter }
} ,
"snapshot" : func ( db ethdb . Database ) utils . ChainDataIterator {
account := db . NewIterator ( rawdb . SnapshotAccountPrefix , nil )
storage := db . NewIterator ( rawdb . SnapshotStoragePrefix , nil )
return & snapshotIterator { account : account , storage : storage }
} ,
}
func exportChaindata ( ctx * cli . Context ) error {
if ctx . NArg ( ) < 2 {
return fmt . Errorf ( "required arguments: %v" , ctx . Command . ArgsUsage )
}
// Parse the required chain data type, make sure it's supported.
kind := ctx . Args ( ) . Get ( 0 )
kind = strings . ToLower ( strings . Trim ( kind , " " ) )
exporter , ok := chainExporters [ kind ]
if ! ok {
var kinds [ ] string
for kind := range chainExporters {
kinds = append ( kinds , kind )
}
return fmt . Errorf ( "invalid data type %s, supported types: %s" , kind , strings . Join ( kinds , ", " ) )
}
var (
stack , _ = makeConfigNode ( ctx )
interrupt = make ( chan os . Signal , 1 )
stop = make ( chan struct { } )
)
defer stack . Close ( )
signal . Notify ( interrupt , syscall . SIGINT , syscall . SIGTERM )
defer signal . Stop ( interrupt )
defer close ( interrupt )
go func ( ) {
if _ , ok := <- interrupt ; ok {
log . Info ( "Interrupted during db export, stopping at next batch" )
}
close ( stop )
} ( )
db := utils . MakeChainDatabase ( ctx , stack , true )
2023-09-25 16:28:20 +00:00
defer db . Close ( )
2021-11-02 10:31:45 +00:00
return utils . ExportChaindata ( ctx . Args ( ) . Get ( 1 ) , kind , exporter ( db ) , stop )
}
2022-01-18 10:30:41 +00:00
func showMetaData ( ctx * cli . Context ) error {
stack , _ := makeConfigNode ( ctx )
defer stack . Close ( )
db := utils . MakeChainDatabase ( ctx , stack , true )
2023-09-25 16:28:20 +00:00
defer db . Close ( )
2022-01-18 10:30:41 +00:00
ancients , err := db . Ancients ( )
if err != nil {
fmt . Fprintf ( os . Stderr , "Error accessing ancients: %v" , err )
}
2023-03-08 07:39:13 +00:00
data := rawdb . ReadChainMetadata ( db )
data = append ( data , [ ] string { "frozen" , fmt . Sprintf ( "%d items" , ancients ) } )
data = append ( data , [ ] string { "snapshotGenerator" , snapshot . ParseGeneratorStatus ( rawdb . ReadSnapshotGenerator ( db ) ) } )
2022-01-18 10:30:41 +00:00
if b := rawdb . ReadHeadBlock ( db ) ; b != nil {
data = append ( data , [ ] string { "headBlock.Hash" , fmt . Sprintf ( "%v" , b . Hash ( ) ) } )
data = append ( data , [ ] string { "headBlock.Root" , fmt . Sprintf ( "%v" , b . Root ( ) ) } )
2022-07-04 08:03:32 +00:00
data = append ( data , [ ] string { "headBlock.Number" , fmt . Sprintf ( "%d (%#x)" , b . Number ( ) , b . Number ( ) ) } )
2022-01-18 10:30:41 +00:00
}
if h := rawdb . ReadHeadHeader ( db ) ; h != nil {
data = append ( data , [ ] string { "headHeader.Hash" , fmt . Sprintf ( "%v" , h . Hash ( ) ) } )
data = append ( data , [ ] string { "headHeader.Root" , fmt . Sprintf ( "%v" , h . Root ) } )
2022-07-04 08:03:32 +00:00
data = append ( data , [ ] string { "headHeader.Number" , fmt . Sprintf ( "%d (%#x)" , h . Number , h . Number ) } )
2022-01-18 10:30:41 +00:00
}
table := tablewriter . NewWriter ( os . Stdout )
table . SetHeader ( [ ] string { "Field" , "Value" } )
table . AppendBulk ( data )
table . Render ( )
return nil
}