cerc-io/plugeth
18
0
Fork 1
Code Issues 4 Pull Requests Packages Projects Releases Wiki Activity

cmd/geth: added counters to the geth inspect report (#21495)

Browse Source 
* database: added counters

* Improved stats for ancient db

* Small improvement

* Better message and added percentage while counting receipts

* Fast counting for receipts

* added info message

* Show both receips itemscount  from ancient db and counted receipts

* Fixed default case

* Removed counter for receipts in ancient store

* Removed counting of receipts present in leveldb
This commit is contained in:
Giuseppe Bertone 2020-09-17 10:23:56 +02:00 committed by GitHub
parent faba018b29
commit 07751c3d26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 121 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
core/rawdb/database.go
View File

@ -238,6 +238,36 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer
return frdb, nil return frdb, nil
} }
type counter uint64
func (c counter) String() string {
return fmt.Sprintf("%d", c)
}
func (c counter) Percentage(current uint64) string {
return fmt.Sprintf("%d", current*100/uint64(c))
}
// stat stores sizes and count for a parameter
type stat struct {
size common.StorageSize
count counter
}
// Add size to the stat and increase the counter by 1
func (s *stat) Add(size common.StorageSize) {
s.size += size
s.count++
}
func (s *stat) Size() string {
return s.size.String()
}
func (s *stat) Count() string {
return s.count.String()
}
// InspectDatabase traverses the entire database and checks the size // InspectDatabase traverses the entire database and checks the size
// of all different categories of data. // of all different categories of data.
func InspectDatabase(db ethdb.Database) error { func InspectDatabase(db ethdb.Database) error {
@ -250,36 +280,38 @@ func InspectDatabase(db ethdb.Database) error {
logged = time.Now() logged = time.Now()
// Key-value store statistics // Key-value store statistics
total common.StorageSize headers stat
headerSize common.StorageSize bodies stat
bodySize common.StorageSize receipts stat
receiptSize common.StorageSize tds stat
tdSize common.StorageSize numHashPairings stat
numHashPairing common.StorageSize hashNumPairings stat
hashNumPairing common.StorageSize tries stat
trieSize common.StorageSize codes stat
codeSize common.StorageSize txLookups stat
txlookupSize common.StorageSize accountSnaps stat
accountSnapSize common.StorageSize storageSnaps stat
storageSnapSize common.StorageSize preimages stat
preimageSize common.StorageSize bloomBits stat
bloomBitsSize common.StorageSize cliqueSnaps stat
cliqueSnapsSize common.StorageSize
// Ancient store statistics // Ancient store statistics
ancientHeaders common.StorageSize ancientHeadersSize common.StorageSize
ancientBodies common.StorageSize ancientBodiesSize common.StorageSize
ancientReceipts common.StorageSize ancientReceiptsSize common.StorageSize
ancientHashes common.StorageSize ancientTdsSize common.StorageSize
ancientTds common.StorageSize ancientHashesSize common.StorageSize
// Les statistic // Les statistic
chtTrieNodes common.StorageSize chtTrieNodes stat
bloomTrieNodes common.StorageSize bloomTrieNodes stat
// Meta- and unaccounted data // Meta- and unaccounted data
metadata common.StorageSize metadata stat
unaccounted common.StorageSize unaccounted stat
// Totals
total common.StorageSize
) )
// Inspect key-value database first. // Inspect key-value database first.
for it.Next() { for it.Next() {
@ -289,98 +321,104 @@ func InspectDatabase(db ethdb.Database) error {
) )
total += size total += size
switch { switch {
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
tdSize += size
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
numHashPairing += size
case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength):
headerSize += size headers.Add(size)
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
hashNumPairing += size
case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength):
bodySize += size bodies.Add(size)
case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
receiptSize += size receipts.Add(size)
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
txlookupSize += size tds.Add(size)
case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
accountSnapSize += size numHashPairings.Add(size)
case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
storageSnapSize += size hashNumPairings.Add(size)
case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength):
preimageSize += size
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
bloomBitsSize += size
case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength:
cliqueSnapsSize += size
case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength:
chtTrieNodes += size
case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength:
bloomTrieNodes += size
case bytes.HasPrefix(key, codePrefix) && len(key) == len(codePrefix)+common.HashLength:
codeSize += size
case len(key) == common.HashLength: case len(key) == common.HashLength:
trieSize += size tries.Add(size)
case bytes.HasPrefix(key, codePrefix) && len(key) == len(codePrefix)+common.HashLength:
codes.Add(size)
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
txLookups.Add(size)
case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength):
accountSnaps.Add(size)
case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength):
storageSnaps.Add(size)
case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength):
preimages.Add(size)
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
bloomBits.Add(size)
case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength:
cliqueSnaps.Add(size)
case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength:
chtTrieNodes.Add(size)
case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength:
bloomTrieNodes.Add(size)
default: default:
var accounted bool var accounted bool
for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} { for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} {
if bytes.Equal(key, meta) { if bytes.Equal(key, meta) {
metadata += size metadata.Add(size)
accounted = true accounted = true
break break
} }
} }
if !accounted { if !accounted {
unaccounted += size unaccounted.Add(size)
} }
} }
count += 1 count++
if count%1000 == 0 && time.Since(logged) > 8*time.Second { if count%1000 == 0 && time.Since(logged) > 8*time.Second {
log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
logged = time.Now() logged = time.Now()
} }
} }
// Inspect append-only file store then. // Inspect append-only file store then.
ancients := []*common.StorageSize{&ancientHeaders, &ancientBodies, &ancientReceipts, &ancientHashes, &ancientTds} ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} {
if size, err := db.AncientSize(category); err == nil { if size, err := db.AncientSize(category); err == nil {
*ancients[i] += common.StorageSize(size) *ancientSizes[i] += common.StorageSize(size)
total += common.StorageSize(size) total += common.StorageSize(size)
} }
} }
// Get number of ancient rows inside the freezer
ancients := counter(0)
if count, err := db.Ancients(); err == nil {
ancients = counter(count)
}
// Display the database statistic. // Display the database statistic.
stats := [][]string{ stats := [][]string{
{"Key-Value store", "Headers", headerSize.String()}, {"Key-Value store", "Headers", headers.Size(), headers.Count()},
{"Key-Value store", "Bodies", bodySize.String()}, {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
{"Key-Value store", "Receipts", receiptSize.String()}, {"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()},
{"Key-Value store", "Difficulties", tdSize.String()}, {"Key-Value store", "Difficulties", tds.Size(), tds.Count()},
{"Key-Value store", "Block number->hash", numHashPairing.String()}, {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()},
{"Key-Value store", "Block hash->number", hashNumPairing.String()}, {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()},
{"Key-Value store", "Transaction index", txlookupSize.String()}, {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
{"Key-Value store", "Bloombit index", bloomBitsSize.String()}, {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
{"Key-Value store", "Contract codes", codeSize.String()}, {"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
{"Key-Value store", "Trie nodes", trieSize.String()}, {"Key-Value store", "Trie nodes", tries.Size(), tries.Count()},
{"Key-Value store", "Trie preimages", preimageSize.String()}, {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
{"Key-Value store", "Account snapshot", accountSnapSize.String()}, {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
{"Key-Value store", "Storage snapshot", storageSnapSize.String()}, {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},
{"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
{"Key-Value store", "Singleton metadata", metadata.String()}, {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
{"Ancient store", "Headers", ancientHeaders.String()}, {"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()},
{"Ancient store", "Bodies", ancientBodies.String()}, {"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()},
{"Ancient store", "Receipts", ancientReceipts.String()}, {"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()},
{"Ancient store", "Difficulties", ancientTds.String()}, {"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()},
{"Ancient store", "Block number->hash", ancientHashes.String()}, {"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()},
{"Light client", "CHT trie nodes", chtTrieNodes.String()}, {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
{"Light client", "Bloom trie nodes", bloomTrieNodes.String()}, {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
} }
table := tablewriter.NewWriter(os.Stdout) table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Database", "Category", "Size"}) table.SetHeader([]string{"Database", "Category", "Size", "Items"})
table.SetFooter([]string{"", "Total", total.String()}) table.SetFooter([]string{"", "Total", total.String(), " "})
table.AppendBulk(stats) table.AppendBulk(stats)
table.Render() table.Render()
if unaccounted > 0 { if unaccounted.size > 0 {
log.Error("Database contains unaccounted data", "size", unaccounted) log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count)
} }
return nil return nil
} }