ethdb: pebble backend (64bit platforms only) (#26517)

* ethdb: use pebble Co-authored-by: Gary Rong <garyrong0905@gmail.com> foo update * apply suggested changes * flags: go format node: fix ddir lookup mistake accounts/abi/bind: fix go.mod replacement for generated binding deps: update pebble + with fix 32-bit build * ethdb/pebble: respect max memtable size * core/rawdb, ethdb: enable pebble on non-32bit platforms only * core/rawdb: fix build tags, fix some review concerns * core/rawdb: refactor methods for database opening * core/rawdb: remove erroneous build tag * cmd/geth: fix the flag default handling + testcase * cmd/geth: improve testing regarding custom backends * ethdb/pebble, deps: update pebble dependency * core/rawdb: replace method with Open * ethdb/pebble: several updates for pebble (#49) * ethdb/pebble: fix size count in batch * ethdb/pebble: disable seek compaction * ethdb/pebble: more fixes * ethdb, core, cmd: polish and fixes (#50) * cmd/utils, core/rawdb, ethdb/pebble: address some review concerns * Update flags.go * ethdb/pebble: minor refactors * ethdb/pebble: avoid copy on batch replay * ethdb: fix compilation flaw * cmd: fix test fail due to mismatching error message * cmd/geth, node: rename backingdb to db.engine --------- Co-authored-by: Jared Wasinger <j-wasinger@hotmail.com> Co-authored-by: rjl493456442 <garyrong0905@gmail.com> Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-02-09 10:48:34 +02:00
parent 095e365fac
commit ed51b8c5d3
16 changed files with 1312 additions and 46 deletions
@@ -17,8 +17,10 @@
 package main

 import (
+	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
 	"testing"
 )

@@ -70,6 +72,7 @@ var customGenesisTests = []struct {
 // Tests that initializing Geth with a custom genesis block and chain definitions
 // work properly.
 func TestCustomGenesis(t *testing.T) {
+	t.Parallel()
 	for i, tt := range customGenesisTests {
 		// Create a temporary data directory to use and inspect later
 		datadir := t.TempDir()
@@ -90,3 +93,101 @@ func TestCustomGenesis(t *testing.T) {
 		geth.ExpectExit()
 	}
 }
+
+// TestCustomBackend that the backend selection and detection (leveldb vs pebble) works properly.
+func TestCustomBackend(t *testing.T) {
+	t.Parallel()
+	// Test pebble, but only on 64-bit platforms
+	if strconv.IntSize != 64 {
+		t.Skip("Custom backends are only available on 64-bit platform")
+	}
+	genesis := `{
+		"alloc"      : {},
+		"coinbase"   : "0x0000000000000000000000000000000000000000",
+			"difficulty" : "0x20000",
+			"extraData"  : "",
+			"gasLimit"   : "0x2fefd8",
+			"nonce"      : "0x0000000000001338",
+			"mixhash"    : "0x0000000000000000000000000000000000000000000000000000000000000000",
+			"parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
+			"timestamp"  : "0x00",
+			"config"     : {}
+	}`
+	type backendTest struct {
+		initArgs   []string
+		initExpect string
+		execArgs   []string
+		execExpect string
+	}
+	testfunc := func(t *testing.T, tt backendTest) error {
+		// Create a temporary data directory to use and inspect later
+		datadir := t.TempDir()
+
+		// Initialize the data directory with the custom genesis block
+		json := filepath.Join(datadir, "genesis.json")
+		if err := os.WriteFile(json, []byte(genesis), 0600); err != nil {
+			return fmt.Errorf("failed to write genesis file: %v", err)
+		}
+		{ // Init
+			args := append(tt.initArgs, "--datadir", datadir, "init", json)
+			geth := runGeth(t, args...)
+			geth.ExpectRegexp(tt.initExpect)
+			geth.ExpectExit()
+		}
+		{ // Exec + query
+			args := append(tt.execArgs, "--networkid", "1337", "--syncmode=full", "--cache", "16",
+				"--datadir", datadir, "--maxpeers", "0", "--port", "0", "--authrpc.port", "0",
+				"--nodiscover", "--nat", "none", "--ipcdisable",
+				"--exec", "eth.getBlock(0).nonce", "console")
+			geth := runGeth(t, args...)
+			geth.ExpectRegexp(tt.execExpect)
+			geth.ExpectExit()
+		}
+		return nil
+	}
+	for i, tt := range []backendTest{
+		{ // When not specified, it should default to leveldb
+			execArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit leveldb
+			initArgs:   []string{"--db.engine", "leveldb"},
+			execArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit leveldb first, then autodiscover
+			initArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit pebble
+			initArgs:   []string{"--db.engine", "pebble"},
+			execArgs:   []string{"--db.engine", "pebble"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit pebble, then auto-discover
+			initArgs:   []string{"--db.engine", "pebble"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Can't start pebble on top of leveldb
+			initArgs:   []string{"--db.engine", "leveldb"},
+			execArgs:   []string{"--db.engine", "pebble"},
+			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
+		},
+		{ // Can't start leveldb on top of pebble
+			initArgs:   []string{"--db.engine", "pebble"},
+			execArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
+		},
+		{ // Reject invalid backend choice
+			initArgs:   []string{"--db.engine", "mssql"},
+			initExpect: `Fatal: Invalid choice for db.engine 'mssql', allowed 'leveldb' or 'pebble'`,
+			// Since the init fails, this will return the (default) mainnet genesis
+			// block nonce
+			execExpect: `0x0000000000000042`,
+		},
+	} {
+		if err := testfunc(t, tt); err != nil {
+			t.Fatalf("test %d-leveldb: %v", i, err)
+		}
+	}
+}
@@ -99,6 +99,12 @@ var (
 		Usage:    "URL for remote database",
 		Category: flags.LoggingCategory,
 	}
+	DBEngineFlag = &cli.StringFlag{
+		Name:     "db.engine",
+		Usage:    "Backing database implementation to use ('leveldb' or 'pebble')",
+		Value:    "leveldb",
+		Category: flags.EthCategory,
+	}
 	AncientFlag = &flags.DirectoryFlag{
 		Name:     "datadir.ancient",
 		Usage:    "Root directory for ancient data (default = inside chaindata)",
@@ -1009,6 +1015,12 @@ var (
 	}
 )

+func init() {
+	if rawdb.PebbleEnabled {
+		DatabasePathFlags = append(DatabasePathFlags, DBEngineFlag)
+	}
+}
+
 // MakeDataDir retrieves the currently requested data directory, terminating
 // if none (or the empty string) is specified. If the node is starting a testnet,
 // then a subdirectory of the specified datadir will be used.
@@ -1484,6 +1496,14 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	if ctx.IsSet(InsecureUnlockAllowedFlag.Name) {
 		cfg.InsecureUnlockAllowed = ctx.Bool(InsecureUnlockAllowedFlag.Name)
 	}
+	if ctx.IsSet(DBEngineFlag.Name) {
+		dbEngine := ctx.String(DBEngineFlag.Name)
+		if dbEngine != "leveldb" && dbEngine != "pebble" {
+			Fatalf("Invalid choice for db.engine '%s', allowed 'leveldb' or 'pebble'", dbEngine)
+		}
+		log.Info(fmt.Sprintf("Using %s as db engine", dbEngine))
+		cfg.DBEngine = dbEngine
+	}
 }

 func setSmartCard(ctx *cli.Context, cfg *node.Config) {