cerc-io/plugeth
12
0
Fork 1
Code Issues 4 Pull Requests Packages Projects Releases Wiki Activity

ethdb: pebble backend (64bit platforms only) (#26517)

Browse Source 
* ethdb: use pebble

Co-authored-by: Gary Rong <garyrong0905@gmail.com>

foo

update

* apply suggested changes

* flags: go format

node: fix ddir lookup mistake

accounts/abi/bind: fix go.mod replacement for generated binding

deps: update pebble + with fix 32-bit build

* ethdb/pebble: respect max memtable size

* core/rawdb, ethdb: enable pebble on non-32bit platforms only

* core/rawdb: fix build tags, fix some review concerns

* core/rawdb: refactor methods for database opening

* core/rawdb: remove erroneous build tag

* cmd/geth: fix the flag default handling + testcase

* cmd/geth: improve testing regarding custom backends

* ethdb/pebble, deps: update pebble dependency

* core/rawdb: replace method with Open

* ethdb/pebble: several updates for pebble (#49)

* ethdb/pebble: fix size count in batch

* ethdb/pebble: disable seek compaction

* ethdb/pebble: more fixes

* ethdb, core, cmd: polish and fixes (#50)

* cmd/utils, core/rawdb, ethdb/pebble: address some review concerns

* Update flags.go

* ethdb/pebble: minor refactors

* ethdb/pebble: avoid copy on batch replay

* ethdb: fix compilation flaw

* cmd: fix test fail due to mismatching error message

* cmd/geth, node: rename backingdb to db.engine

---------

Co-authored-by: Jared Wasinger <j-wasinger@hotmail.com>
Co-authored-by: rjl493456442 <garyrong0905@gmail.com>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
Martin Holst Swende
2023-02-09 10:48:34 +02:00
committed by GitHub
co-authored by Gary Rong Jared Wasinger Péter Szilágyi
parent 095e365fac
commit ed51b8c5d3
16 changed files with 1312 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/rawdb/database.go
+73 -7
View File
@@ -22,6 +22,7 @@ import (
"fmt"
"os"
"path"
"path/filepath"
"strings"
"sync/atomic"
"time"
@@ -302,19 +303,84 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r
if err != nil {
return nil, err
}
log.Info("Using LevelDB as the backing database")
return NewDatabase(db), nil
}
// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
// freezer moving immutable chain segments into cold storage. The passed ancient
// indicates the path of root ancient directory where the chain freezer can be
// opened.
func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) {
kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
const (
dbPebble = "pebble"
dbLeveldb = "leveldb"
)
// hasPreexistingDb checks the given data directory whether a database is already
// instantiated at that location, and if so, returns the type of database (or the
// empty string).
func hasPreexistingDb(path string) string {
if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
return "" // No pre-existing db
}
if matches, err := filepath.Glob(filepath.Join(path, "OPTIONS*")); len(matches) > 0 || err != nil {
if err != nil {
panic(err) // only possible if the pattern is malformed
}
return dbPebble
}
return dbLeveldb
}
// OpenOptions contains the options to apply when opening a database.
// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used.
type OpenOptions struct {
Type string // "leveldb" | "pebble"
Directory string // the datadir
AncientsDirectory string // the ancients-dir
Namespace string // the namespace for database relevant metrics
Cache int // the capacity(in megabytes) of the data caching
Handles int // number of files to be open simultaneously
ReadOnly bool
}
// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
//
// type == null type != null
// +----------------------------------------
// db is non-existent | leveldb default | specified type
// db is existent | from db | specified type (if compatible)
func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
existingDb := hasPreexistingDb(o.Directory)
if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb {
return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
}
if o.Type == dbPebble || existingDb == dbPebble {
if PebbleEnabled {
log.Info("Using pebble as the backing database")
return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
} else {
return nil, errors.New("db.engine 'pebble' not supported on this platform")
}
}
if len(o.Type) != 0 && o.Type != dbLeveldb {
return nil, fmt.Errorf("unknown db.engine %v", o.Type)
}
log.Info("Using leveldb as the backing database")
// Use leveldb, either as default (no explicit choice), or pre-existing, or chosen explicitly
return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
// Open opens both a disk-based key-value database such as leveldb or pebble, but also
// integrates it with a freezer database -- if the AncientDir option has been
// set on the provided OpenOptions.
// The passed o.AncientDir indicates the path of root ancient directory where
// the chain freezer can be opened.
func Open(o OpenOptions) (ethdb.Database, error) {
kvdb, err := openKeyValueDatabase(o)
if err != nil {
return nil, err
}
frdb, err := NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly)
if len(o.AncientsDirectory) == 0 {
return kvdb, nil
}
frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly)
if err != nil {
kvdb.Close()
return nil, err
core/rawdb/databases_64bit.go
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
//go:build arm64 || amd64
package rawdb
import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/pebble"
)
// Pebble is unsuported on 32bit architecture
const PebbleEnabled = true
// NewPebbleDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
db, err := pebble.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
return NewDatabase(db), nil
}
core/rawdb/databases_non64bit.go
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
//go:build !(arm64 || amd64)
package rawdb
import (
"errors"
"github.com/ethereum/go-ethereum/ethdb"
)
// Pebble is unsuported on 32bit architecture
const PebbleEnabled = false
// NewPebbleDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
return nil, errors.New("pebble is not supported on this platform")
}