From c7e7778f2a7d80fa12643db546db98fa70f2e384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 22 Jul 2015 13:46:20 +0300 Subject: [PATCH] cmd, core, eth, ethdb: cache flag to allocate memory for db internal use --- cmd/geth/main.go | 3 ++- cmd/utils/flags.go | 16 ++++++++++++---- core/bench_test.go | 2 +- eth/backend.go | 3 ++- ethdb/database.go | 31 +++++++++++++++++++++++++------ ethdb/database_test.go | 3 +-- 6 files changed, 43 insertions(+), 15 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 8099d4dc6..f20ea390f 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -281,6 +281,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso utils.BootnodesFlag, utils.DataDirFlag, utils.BlockchainVersionFlag, + utils.CacheFlag, utils.JSpathFlag, utils.ListenPortFlag, utils.MaxPeersFlag, @@ -492,7 +493,7 @@ func blockRecovery(ctx *cli.Context) { cfg := utils.MakeEthConfig(ClientIdentifier, nodeNameVersion, ctx) utils.CheckLegalese(cfg.DataDir) - blockDb, err := ethdb.NewLDBDatabase(filepath.Join(cfg.DataDir, "blockchain")) + blockDb, err := ethdb.NewLDBDatabase(filepath.Join(cfg.DataDir, "blockchain"), cfg.DatabaseCache) if err != nil { glog.Fatalln("could not open db:", err) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 73bdb935a..68de67cde 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -126,6 +126,11 @@ var ( Name: "natspec", Usage: "Enable NatSpec confirmation notice", } + CacheFlag = cli.IntFlag{ + Name: "cache", + Usage: "Megabytes of memory allocated to internal caching", + Value: 0, + } // miner settings MinerThreadsFlag = cli.IntFlag{ @@ -384,6 +389,7 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { GenesisNonce: ctx.GlobalInt(GenesisNonceFlag.Name), GenesisFile: ctx.GlobalString(GenesisFileFlag.Name), BlockChainVersion: ctx.GlobalInt(BlockchainVersionFlag.Name), + DatabaseCache: ctx.GlobalInt(CacheFlag.Name), SkipBcVersionCheck: false, NetworkId: ctx.GlobalInt(NetworkIdFlag.Name), LogFile: ctx.GlobalString(LogFileFlag.Name), @@ -425,15 +431,17 @@ func SetupLogger(ctx *cli.Context) { // MakeChain creates a chain manager from set command line flags. func MakeChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, extraDB common.Database) { - dd := ctx.GlobalString(DataDirFlag.Name) + datadir := ctx.GlobalString(DataDirFlag.Name) + cache := ctx.GlobalInt(CacheFlag.Name) + var err error - if blockDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "blockchain")); err != nil { + if blockDB, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "blockchain"), cache); err != nil { Fatalf("Could not open database: %v", err) } - if stateDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "state")); err != nil { + if stateDB, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "state"), cache); err != nil { Fatalf("Could not open database: %v", err) } - if extraDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "extra")); err != nil { + if extraDB, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "extra"), cache); err != nil { Fatalf("Could not open database: %v", err) } diff --git a/core/bench_test.go b/core/bench_test.go index 018d27d8d..ec0474ad9 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -153,7 +153,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { b.Fatalf("cannot create temporary directory: %v", err) } defer os.RemoveAll(dir) - db, err = ethdb.NewLDBDatabase(dir) + db, err = ethdb.NewLDBDatabase(dir, 0) if err != nil { b.Fatalf("cannot create temporary database: %v", err) } diff --git a/eth/backend.go b/eth/backend.go index e7250c019..4906f78ef 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -80,6 +80,7 @@ type Config struct { BlockChainVersion int SkipBcVersionCheck bool // e.g. blockchain export + DatabaseCache int DataDir string LogFile string @@ -261,7 +262,7 @@ func New(config *Config) (*Ethereum, error) { newdb := config.NewDB if newdb == nil { - newdb = func(path string) (common.Database, error) { return ethdb.NewLDBDatabase(path) } + newdb = func(path string) (common.Database, error) { return ethdb.NewLDBDatabase(path, config.DatabaseCache) } } blockDb, err := newdb(filepath.Join(config.DataDir, "blockchain")) if err != nil { diff --git a/ethdb/database.go b/ethdb/database.go index c75136a1b..38e454c00 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -17,6 +17,7 @@ package ethdb import ( + "path/filepath" "strconv" "strings" "sync" @@ -36,6 +37,14 @@ import ( var OpenFileLimit = 64 +// cacheRatio specifies how the total alloted cache is distributed between the +// various system databases. +var cacheRatio = map[string]float64{ + "blockchain": 1.0 / 13.0, + "extra": 2.0 / 13.0, + "state": 10.0 / 13.0, +} + type LDBDatabase struct { fn string // filename for reporting db *leveldb.DB // LevelDB instance @@ -57,14 +66,24 @@ type LDBDatabase struct { // NewLDBDatabase returns a LevelDB wrapped object. LDBDatabase does not persist data by // it self but requires a background poller which syncs every X. `Flush` should be called // when data needs to be stored and written to disk. -func NewLDBDatabase(file string) (*LDBDatabase, error) { - // Open the db - db, err := leveldb.OpenFile(file, &opt.Options{OpenFilesCacheCapacity: OpenFileLimit}) - // check for corruption and attempt to recover - if _, iscorrupted := err.(*errors.ErrCorrupted); iscorrupted { +func NewLDBDatabase(file string, cache int) (*LDBDatabase, error) { + // Calculate the cache allowance for this particular database + cache = int(float64(cache) * cacheRatio[filepath.Base(file)]) + if cache < 16 { + cache = 16 + } + glog.V(logger.Info).Infof("Alloted %dMB cache to %s", cache, file) + + // Open the db and recover any potential corruptions + db, err := leveldb.OpenFile(file, &opt.Options{ + OpenFilesCacheCapacity: OpenFileLimit, + BlockCacheCapacity: cache / 2 * opt.MiB, + WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally + }) + if _, corrupted := err.(*errors.ErrCorrupted); corrupted { db, err = leveldb.RecoverFile(file, nil) } - // (re) check for errors and abort if opening of the db failed + // (Re)check for errors and abort if opening of the db failed if err != nil { return nil, err } diff --git a/ethdb/database_test.go b/ethdb/database_test.go index 29292d016..41947a698 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -28,8 +28,7 @@ func newDb() *LDBDatabase { if common.FileExist(file) { os.RemoveAll(file) } - - db, _ := NewLDBDatabase(file) + db, _ := NewLDBDatabase(file, 0) return db }