core: recover state when beacon sets canonical head if it's missing (#24613)

* core: recover the state in SetChainHead if the head state is missing

* core: disable test logging

* core: address comment from martin

* core: improve log level in case state is recovered

* core, eth, les, light: rename SetChainHead to SetCanonical

core/blockchain.go

@@ -2075,7 +2075,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 // InsertBlockWithoutSetHead executes the block, runs the necessary verification
 // upon it and then persist the block and the associate state into the database.
 // The key difference between the InsertChain is it won't do the canonical chain
-// updating. It relies on the additional SetChainHead call to finalize the entire
+// updating. It relies on the additional SetCanonical call to finalize the entire
 // procedure.
 func (bc *BlockChain) InsertBlockWithoutSetHead(block *types.Block) error {
 	if !bc.chainmu.TryLock() {
@@ -2087,16 +2087,22 @@ func (bc *BlockChain) InsertBlockWithoutSetHead(block *types.Block) error {
 	return err
 }
 
-// SetChainHead rewinds the chain to set the new head block as the specified
+// SetCanonical rewinds the chain to set the new head block as the specified
-// block. It's possible that after the reorg the relevant state of head
+// block. It's possible that the state of the new head is missing, and it will
-// is missing. It can be fixed by inserting a new block which triggers
+// be recovered in this function as well.
-// the re-execution.
+func (bc *BlockChain) SetCanonical(head *types.Block) error {
-func (bc *BlockChain) SetChainHead(head *types.Block) error {
 	if !bc.chainmu.TryLock() {
 		return errChainStopped
 	}
 	defer bc.chainmu.Unlock()
 
+	// Re-execute the reorged chain in case the head state is missing.
+	if !bc.HasState(head.Root()) {
+		if err := bc.recoverAncestors(head); err != nil {
+			return err
+		}
+		log.Info("Recovered head state", "number", head.Number(), "hash", head.Hash())
+	}
 	// Run the reorg if necessary and set the given block as new head.
 	start := time.Now()
 	if head.ParentHash() != bc.CurrentBlock().Hash() {

core/blockchain_test.go

@@ -3676,3 +3676,85 @@ func TestEIP1559Transition(t *testing.T) {
 		t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual)
 	}
 }
+
+// Tests the scenario the chain is requested to another point with the missing state.
+// It expects the state is recovered and all relevant chain markers are set correctly.
+func TestSetCanonical(t *testing.T) {
+	//log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
+
+	var (
+		db      = rawdb.NewMemoryDatabase()
+		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		address = crypto.PubkeyToAddress(key.PublicKey)
+		funds   = big.NewInt(100000000000000000)
+		gspec   = &Genesis{
+			Config:  params.TestChainConfig,
+			Alloc:   GenesisAlloc{address: {Balance: funds}},
+			BaseFee: big.NewInt(params.InitialBaseFee),
+		}
+		genesis = gspec.MustCommit(db)
+		signer  = types.LatestSigner(gspec.Config)
+		engine  = ethash.NewFaker()
+	)
+	// Generate and import the canonical chain
+	canon, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, func(i int, gen *BlockGen) {
+		tx, err := types.SignTx(types.NewTransaction(gen.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, gen.header.BaseFee, nil), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		gen.AddTx(tx)
+	})
+	diskdb := rawdb.NewMemoryDatabase()
+	gspec.MustCommit(diskdb)
+
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil)
+	if err != nil {
+		t.Fatalf("failed to create tester chain: %v", err)
+	}
+	if n, err := chain.InsertChain(canon); err != nil {
+		t.Fatalf("block %d: failed to insert into chain: %v", n, err)
+	}
+
+	// Generate the side chain and import them
+	side, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, func(i int, gen *BlockGen) {
+		tx, err := types.SignTx(types.NewTransaction(gen.TxNonce(address), common.Address{0x00}, big.NewInt(1), params.TxGas, gen.header.BaseFee, nil), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		gen.AddTx(tx)
+	})
+	for _, block := range side {
+		err := chain.InsertBlockWithoutSetHead(block)
+		if err != nil {
+			t.Fatalf("Failed to insert into chain: %v", err)
+		}
+	}
+	for _, block := range side {
+		got := chain.GetBlockByHash(block.Hash())
+		if got == nil {
+			t.Fatalf("Lost the inserted block")
+		}
+	}
+
+	// Set the chain head to the side chain, ensure all the relevant markers are updated.
+	verify := func(head *types.Block) {
+		if chain.CurrentBlock().Hash() != head.Hash() {
+			t.Fatalf("Unexpected block hash, want %x, got %x", head.Hash(), chain.CurrentBlock().Hash())
+		}
+		if chain.CurrentFastBlock().Hash() != head.Hash() {
+			t.Fatalf("Unexpected fast block hash, want %x, got %x", head.Hash(), chain.CurrentFastBlock().Hash())
+		}
+		if chain.CurrentHeader().Hash() != head.Hash() {
+			t.Fatalf("Unexpected head header, want %x, got %x", head.Hash(), chain.CurrentHeader().Hash())
+		}
+		if !chain.HasState(head.Root()) {
+			t.Fatalf("Lost block state %v %x", head.Number(), head.Hash())
+		}
+	}
+	chain.SetCanonical(side[len(side)-1])
+	verify(side[len(side)-1])
+
+	// Reset the chain head to original chain
+	chain.SetCanonical(canon[TriesInMemory-1])
+	verify(canon[TriesInMemory-1])
+}

core/headerchain.go

@@ -204,7 +204,7 @@ func (hc *HeaderChain) Reorg(headers []*types.Header) error {
 
 // WriteHeaders writes a chain of headers into the local chain, given that the
 // parents are already known. The chain head header won't be updated in this
-// function, the additional setChainHead is expected in order to finish the entire
+// function, the additional SetCanonical is expected in order to finish the entire
 // procedure.
 func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) {
 	if len(headers) == 0 {

eth/catalyst/api.go

@@ -139,7 +139,7 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa
 
 	if rawdb.ReadCanonicalHash(api.eth.ChainDb(), block.NumberU64()) != update.HeadBlockHash {
 		// Block is not canonical, set head.
-		if err := api.eth.BlockChain().SetChainHead(block); err != nil {
+		if err := api.eth.BlockChain().SetCanonical(block); err != nil {
 			return beacon.STATUS_INVALID, err
 		}
 	} else {

les/catalyst/api.go

@@ -87,7 +87,7 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(heads beacon.ForkchoiceStateV1, pay
 		}
 	}
 	// SetHead
-	if err := api.setHead(heads.HeadBlockHash); err != nil {
+	if err := api.setCanonical(heads.HeadBlockHash); err != nil {
 		return beacon.STATUS_INVALID, err
 	}
 	if payloadAttributes != nil {
@@ -166,8 +166,8 @@ func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error {
 	return nil
 }
 
-// setHead is called to perform a force choice.
+// setCanonical is called to perform a force choice.
-func (api *ConsensusAPI) setHead(newHead common.Hash) error {
+func (api *ConsensusAPI) setCanonical(newHead common.Hash) error {
 	log.Info("Setting head", "head", newHead)
 
 	headHeader := api.les.BlockChain().CurrentHeader()
@@ -178,7 +178,7 @@ func (api *ConsensusAPI) setHead(newHead common.Hash) error {
 	if newHeadHeader == nil {
 		return &beacon.GenericServerError
 	}
-	if err := api.les.BlockChain().SetChainHead(newHeadHeader); err != nil {
+	if err := api.les.BlockChain().SetCanonical(newHeadHeader); err != nil {
 		return err
 	}
 	// Trigger the transition if it's the first `NewHead` event.

les/catalyst/api_test.go

@@ -194,7 +194,7 @@ func TestEth2DeepReorg(t *testing.T) {
 			if ethservice.BlockChain().CurrentBlock().NumberU64() != head {
 				t.Fatalf("Chain head shouldn't be updated")
 			}
-			if err := api.setHead(block.Hash()); err != nil {
+			if err := api.setCanonical(block.Hash()); err != nil {
 				t.Fatalf("Failed to set head: %v", err)
 			}
 			if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() {

light/lightchain.go

@@ -389,7 +389,7 @@ func (lc *LightChain) InsertHeader(header *types.Header) error {
 	return err
 }
 
-func (lc *LightChain) SetChainHead(header *types.Header) error {
+func (lc *LightChain) SetCanonical(header *types.Header) error {
 	lc.chainmu.Lock()
 	defer lc.chainmu.Unlock()