cmd/geth, consensus/ethash: add support for --miner.notify.full flag (#22558)

The PR implements the --miner.notify.full flag that enables full pending block
notifications. When this flag is used, the block notifications sent to mining
endpoints contain the complete block header JSON instead of a work package
array.

Co-authored-by: AlexSSD7 <alexandersadovskyi7@protonmail.com>
Co-authored-by: Martin Holst Swende <martin@swende.se>
This commit is contained in:
Felix Lange 2021-03-26 18:30:10 +01:00 committed by GitHub
parent 955727181b
commit cae6b5527e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 171 additions and 39 deletions

View File

@ -152,6 +152,7 @@ var (
utils.GpoMaxGasPriceFlag, utils.GpoMaxGasPriceFlag,
utils.EWASMInterpreterFlag, utils.EWASMInterpreterFlag,
utils.EVMInterpreterFlag, utils.EVMInterpreterFlag,
utils.MinerNotifyFullFlag,
configFileFlag, configFileFlag,
} }

View File

@ -180,6 +180,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
utils.MiningEnabledFlag, utils.MiningEnabledFlag,
utils.MinerThreadsFlag, utils.MinerThreadsFlag,
utils.MinerNotifyFlag, utils.MinerNotifyFlag,
utils.MinerNotifyFullFlag,
utils.MinerGasPriceFlag, utils.MinerGasPriceFlag,
utils.MinerGasTargetFlag, utils.MinerGasTargetFlag,
utils.MinerGasLimitFlag, utils.MinerGasLimitFlag,

View File

@ -427,6 +427,10 @@ var (
Name: "miner.notify", Name: "miner.notify",
Usage: "Comma separated HTTP URL list to notify of new work packages", Usage: "Comma separated HTTP URL list to notify of new work packages",
} }
MinerNotifyFullFlag = cli.BoolFlag{
Name: "miner.notify.full",
Usage: "Notify with pending block headers instead of work packages",
}
MinerGasTargetFlag = cli.Uint64Flag{ MinerGasTargetFlag = cli.Uint64Flag{
Name: "miner.gastarget", Name: "miner.gastarget",
Usage: "Target gas floor for mined blocks", Usage: "Target gas floor for mined blocks",
@ -1359,6 +1363,7 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) {
if ctx.GlobalIsSet(MinerNotifyFlag.Name) { if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
cfg.Notify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") cfg.Notify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
} }
cfg.NotifyFull = ctx.GlobalBool(MinerNotifyFullFlag.Name)
if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { if ctx.GlobalIsSet(MinerExtraDataFlag.Name) {
cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name))
} }

View File

@ -726,10 +726,14 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
pend.Add(1) pend.Add(1)
go func(idx int) { go func(idx int) {
defer pend.Done() defer pend.Done()
ethash := New(Config{cachedir, 0, 1, false, "", 0, 0, false, ModeNormal, nil}, nil, false)
config := Config{
CacheDir: cachedir,
CachesOnDisk: 1,
}
ethash := New(config, nil, false)
defer ethash.Close() defer ethash.Close()
if err := ethash.verifySeal(nil, block.Header(), false); err != nil { if err := ethash.verifySeal(nil, block.Header(), false); err != nil {
t.Errorf("proc %d: block verification failed: %v", idx, err) t.Errorf("proc %d: block verification failed: %v", idx, err)

View File

@ -48,7 +48,7 @@ var (
two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
// sharedEthash is a full instance that can be shared between multiple users. // sharedEthash is a full instance that can be shared between multiple users.
sharedEthash = New(Config{"", 3, 0, false, "", 1, 0, false, ModeNormal, nil}, nil, false) sharedEthash *Ethash
// algorithmRevision is the data structure version used for file naming. // algorithmRevision is the data structure version used for file naming.
algorithmRevision = 23 algorithmRevision = 23
@ -57,6 +57,15 @@ var (
dumpMagic = []uint32{0xbaddcafe, 0xfee1dead} dumpMagic = []uint32{0xbaddcafe, 0xfee1dead}
) )
func init() {
sharedConfig := Config{
PowMode: ModeNormal,
CachesInMem: 3,
DatasetsInMem: 1,
}
sharedEthash = New(sharedConfig, nil, false)
}
// isLittleEndian returns whether the local system is running in little or big // isLittleEndian returns whether the local system is running in little or big
// endian byte order. // endian byte order.
func isLittleEndian() bool { func isLittleEndian() bool {
@ -411,6 +420,10 @@ type Config struct {
DatasetsLockMmap bool DatasetsLockMmap bool
PowMode Mode PowMode Mode
// When set, notifications sent by the remote sealer will
// be block header JSON objects instead of work package arrays.
NotifyFull bool
Log log.Logger `toml:"-"` Log log.Logger `toml:"-"`
} }
@ -462,6 +475,9 @@ func New(config Config, notify []string, noverify bool) *Ethash {
update: make(chan struct{}), update: make(chan struct{}),
hashrate: metrics.NewMeterForced(), hashrate: metrics.NewMeterForced(),
} }
if config.PowMode == ModeShared {
ethash.shared = sharedEthash
}
ethash.remote = startRemoteSealer(ethash, notify, noverify) ethash.remote = startRemoteSealer(ethash, notify, noverify)
return ethash return ethash
} }
@ -469,15 +485,7 @@ func New(config Config, notify []string, noverify bool) *Ethash {
// NewTester creates a small sized ethash PoW scheme useful only for testing // NewTester creates a small sized ethash PoW scheme useful only for testing
// purposes. // purposes.
func NewTester(notify []string, noverify bool) *Ethash { func NewTester(notify []string, noverify bool) *Ethash {
ethash := &Ethash{ return New(Config{PowMode: ModeTest}, notify, noverify)
config: Config{PowMode: ModeTest, Log: log.Root()},
caches: newlru("cache", 1, newCache),
datasets: newlru("dataset", 1, newDataset),
update: make(chan struct{}),
hashrate: metrics.NewMeterForced(),
}
ethash.remote = startRemoteSealer(ethash, notify, noverify)
return ethash
} }
// NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts // NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts

View File

@ -62,7 +62,14 @@ func TestCacheFileEvict(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(tmpdir) defer os.RemoveAll(tmpdir)
e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil, false)
config := Config{
CachesInMem: 3,
CachesOnDisk: 10,
CacheDir: tmpdir,
PowMode: ModeTest,
}
e := New(config, nil, false)
defer e.Close() defer e.Close()
workers := 8 workers := 8

View File

@ -358,7 +358,16 @@ func (s *remoteSealer) makeWork(block *types.Block) {
// new work to be processed. // new work to be processed.
func (s *remoteSealer) notifyWork() { func (s *remoteSealer) notifyWork() {
work := s.currentWork work := s.currentWork
blob, _ := json.Marshal(work)
// Encode the JSON payload of the notification. When NotifyFull is set,
// this is the complete block header, otherwise it is a JSON array.
var blob []byte
if s.ethash.config.NotifyFull {
blob, _ = json.Marshal(s.currentBlock.Header())
} else {
blob, _ = json.Marshal(work)
}
s.reqWG.Add(len(s.notifyURLs)) s.reqWG.Add(len(s.notifyURLs))
for _, url := range s.notifyURLs { for _, url := range s.notifyURLs {
go s.sendNotification(s.notifyCtx, url, blob, work) go s.sendNotification(s.notifyCtx, url, blob, work)

View File

@ -22,6 +22,7 @@ import (
"math/big" "math/big"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strconv"
"testing" "testing"
"time" "time"
@ -74,6 +75,50 @@ func TestRemoteNotify(t *testing.T) {
} }
} }
// Tests whether remote HTTP servers are correctly notified of new work. (Full pending block body / --miner.notify.full)
func TestRemoteNotifyFull(t *testing.T) {
// Start a simple web server to capture notifications.
sink := make(chan map[string]interface{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
blob, err := ioutil.ReadAll(req.Body)
if err != nil {
t.Errorf("failed to read miner notification: %v", err)
}
var work map[string]interface{}
if err := json.Unmarshal(blob, &work); err != nil {
t.Errorf("failed to unmarshal miner notification: %v", err)
}
sink <- work
}))
defer server.Close()
// Create the custom ethash engine.
config := Config{
PowMode: ModeTest,
NotifyFull: true,
Log: testlog.Logger(t, log.LvlWarn),
}
ethash := New(config, []string{server.URL}, false)
defer ethash.Close()
// Stream a work task and ensure the notification bubbles out.
header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
block := types.NewBlockWithHeader(header)
ethash.Seal(nil, block, nil, nil)
select {
case work := <-sink:
if want := "0x" + strconv.FormatUint(header.Number.Uint64(), 16); work["number"] != want {
t.Errorf("pending block number mismatch: have %v, want %v", work["number"], want)
}
if want := "0x" + header.Difficulty.Text(16); work["difficulty"] != want {
t.Errorf("pending block difficulty mismatch: have %s, want %s", work["difficulty"], want)
}
case <-time.After(3 * time.Second):
t.Fatalf("notification timed out")
}
}
// Tests that pushing work packages fast to the miner doesn't cause any data race // Tests that pushing work packages fast to the miner doesn't cause any data race
// issues in the notifications. // issues in the notifications.
func TestRemoteMultiNotify(t *testing.T) { func TestRemoteMultiNotify(t *testing.T) {
@ -119,6 +164,55 @@ func TestRemoteMultiNotify(t *testing.T) {
} }
} }
// Tests that pushing work packages fast to the miner doesn't cause any data race
// issues in the notifications. Full pending block body / --miner.notify.full)
func TestRemoteMultiNotifyFull(t *testing.T) {
// Start a simple web server to capture notifications.
sink := make(chan map[string]interface{}, 64)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
blob, err := ioutil.ReadAll(req.Body)
if err != nil {
t.Errorf("failed to read miner notification: %v", err)
}
var work map[string]interface{}
if err := json.Unmarshal(blob, &work); err != nil {
t.Errorf("failed to unmarshal miner notification: %v", err)
}
sink <- work
}))
defer server.Close()
// Create the custom ethash engine.
config := Config{
PowMode: ModeTest,
NotifyFull: true,
Log: testlog.Logger(t, log.LvlWarn),
}
ethash := New(config, []string{server.URL}, false)
defer ethash.Close()
// Provide a results reader.
// Otherwise the unread results will be logged asynchronously
// and this can happen after the test is finished, causing a panic.
results := make(chan *types.Block, cap(sink))
// Stream a lot of work task and ensure all the notifications bubble out.
for i := 0; i < cap(sink); i++ {
header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)}
block := types.NewBlockWithHeader(header)
ethash.Seal(nil, block, results, nil)
}
for i := 0; i < cap(sink); i++ {
select {
case <-sink:
<-results
case <-time.After(10 * time.Second):
t.Fatalf("notification %d timed out", i)
}
}
}
// Tests whether stale solutions are correctly processed. // Tests whether stale solutions are correctly processed.
func TestStaleSubmission(t *testing.T) { func TestStaleSubmission(t *testing.T) {
ethash := NewTester(nil, true) ethash := NewTester(nil, true)

View File

@ -121,6 +121,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
} }
log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
// Transfer mining-related config to the ethash config.
ethashConfig := config.Ethash
ethashConfig.NotifyFull = config.Miner.NotifyFull
// Assemble the Ethereum object // Assemble the Ethereum object
chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false) chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)
if err != nil { if err != nil {
@ -140,7 +144,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
chainDb: chainDb, chainDb: chainDb,
eventMux: stack.EventMux(), eventMux: stack.EventMux(),
accountManager: stack.AccountManager(), accountManager: stack.AccountManager(),
engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, config.Miner.Notify, config.Miner.Noverify, chainDb), engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &ethashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb),
closeBloomHandler: make(chan struct{}), closeBloomHandler: make(chan struct{}),
networkID: config.NetworkId, networkID: config.NetworkId,
gasPrice: config.Miner.GasPrice, gasPrice: config.Miner.GasPrice,

View File

@ -213,25 +213,23 @@ func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, co
switch config.PowMode { switch config.PowMode {
case ethash.ModeFake: case ethash.ModeFake:
log.Warn("Ethash used in fake mode") log.Warn("Ethash used in fake mode")
return ethash.NewFaker()
case ethash.ModeTest: case ethash.ModeTest:
log.Warn("Ethash used in test mode") log.Warn("Ethash used in test mode")
return ethash.NewTester(nil, noverify)
case ethash.ModeShared: case ethash.ModeShared:
log.Warn("Ethash used in shared mode") log.Warn("Ethash used in shared mode")
return ethash.NewShared()
default:
engine := ethash.New(ethash.Config{
CacheDir: stack.ResolvePath(config.CacheDir),
CachesInMem: config.CachesInMem,
CachesOnDisk: config.CachesOnDisk,
CachesLockMmap: config.CachesLockMmap,
DatasetDir: config.DatasetDir,
DatasetsInMem: config.DatasetsInMem,
DatasetsOnDisk: config.DatasetsOnDisk,
DatasetsLockMmap: config.DatasetsLockMmap,
}, notify, noverify)
engine.SetThreads(-1) // Disable CPU mining
return engine
} }
engine := ethash.New(ethash.Config{
PowMode: config.PowMode,
CacheDir: stack.ResolvePath(config.CacheDir),
CachesInMem: config.CachesInMem,
CachesOnDisk: config.CachesOnDisk,
CachesLockMmap: config.CachesLockMmap,
DatasetDir: config.DatasetDir,
DatasetsInMem: config.DatasetsInMem,
DatasetsOnDisk: config.DatasetsOnDisk,
DatasetsLockMmap: config.DatasetsLockMmap,
NotifyFull: config.NotifyFull,
}, notify, noverify)
engine.SetThreads(-1) // Disable CPU mining
return engine
} }

View File

@ -42,14 +42,15 @@ type Backend interface {
// Config is the configuration parameters of mining. // Config is the configuration parameters of mining.
type Config struct { type Config struct {
Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards (default = first account) Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards (default = first account)
Notify []string `toml:",omitempty"` // HTTP URL list to be notified of new work packages(only useful in ethash). Notify []string `toml:",omitempty"` // HTTP URL list to be notified of new work packages (only useful in ethash).
ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner NotifyFull bool `toml:",omitempty"` // Notify with pending block headers instead of work packages
GasFloor uint64 // Target gas floor for mined blocks. ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner
GasCeil uint64 // Target gas ceiling for mined blocks. GasFloor uint64 // Target gas floor for mined blocks.
GasPrice *big.Int // Minimum gas price for mining a transaction GasCeil uint64 // Target gas ceiling for mined blocks.
Recommit time.Duration // The time interval for miner to re-create mining work. GasPrice *big.Int // Minimum gas price for mining a transaction
Noverify bool // Disable remote mining solution verification(only useful in ethash). Recommit time.Duration // The time interval for miner to re-create mining work.
Noverify bool // Disable remote mining solution verification(only useful in ethash).
} }
// Miner creates blocks and searches for proof-of-work values. // Miner creates blocks and searches for proof-of-work values.