Merge pull request #10979 from filecoin-project/asr/slasher-improvements

feat: daemon: improvemens to the consensus slasher

chain/gen/slashfilter/slashfilter.go

@@ -26,20 +26,30 @@ func New(dstore ds.Batching) *SlashFilter {
 	}
 }
 
-func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, parentEpoch abi.ChainEpoch) (cid.Cid, error) {
+func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, parentEpoch abi.ChainEpoch) (cid.Cid, bool, error) {
 	epochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, bh.Height))
 	{
 		// double-fork mining (2 blocks at one epoch)
-		if witness, err := checkFault(ctx, f.byEpoch, epochKey, bh, "double-fork mining faults"); err != nil {
+		doubleForkWitness, doubleForkFault, err := checkFault(ctx, f.byEpoch, epochKey, bh, "double-fork mining faults")
-			return witness, xerrors.Errorf("check double-fork mining faults: %w", err)
+		if err != nil {
+			return cid.Undef, false, xerrors.Errorf("check double-fork mining faults: %w", err)
+		}
+
+		if doubleForkFault {
+			return doubleForkWitness, doubleForkFault, nil
 		}
 	}
 
 	parentsKey := ds.NewKey(fmt.Sprintf("/%s/%x", bh.Miner, types.NewTipSetKey(bh.Parents...).Bytes()))
 	{
 		// time-offset mining faults (2 blocks with the same parents)
-		if witness, err := checkFault(ctx, f.byParents, parentsKey, bh, "time-offset mining faults"); err != nil {
+		timeOffsetWitness, timeOffsetFault, err := checkFault(ctx, f.byParents, parentsKey, bh, "time-offset mining faults")
-			return witness, xerrors.Errorf("check time-offset mining faults: %w", err)
+		if err != nil {
+			return cid.Undef, false, xerrors.Errorf("check time-offset mining faults: %w", err)
+		}
+
+		if timeOffsetFault {
+			return timeOffsetWitness, timeOffsetFault, nil
 		}
 	}
 
@@ -50,19 +60,19 @@ func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, par
 		parentEpochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, parentEpoch))
 		have, err := f.byEpoch.Has(ctx, parentEpochKey)
 		if err != nil {
-			return cid.Undef, err
+			return cid.Undef, false, xerrors.Errorf("failed to read from db: %w", err)
 		}
 
 		if have {
 			// If we had, make sure it's in our parent tipset
 			cidb, err := f.byEpoch.Get(ctx, parentEpochKey)
 			if err != nil {
-				return cid.Undef, xerrors.Errorf("getting other block cid: %w", err)
+				return cid.Undef, false, xerrors.Errorf("getting other block cid: %w", err)
 			}
 
 			_, parent, err := cid.CidFromBytes(cidb)
 			if err != nil {
-				return cid.Undef, err
+				return cid.Undef, false, xerrors.Errorf("failed to read cid from bytes: %w", err)
 			}
 
 			var found bool
@@ -73,45 +83,45 @@ func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, par
 			}
 
 			if !found {
-				return parent, xerrors.Errorf("produced block would trigger 'parent-grinding fault' consensus fault; miner: %s; bh: %s, expected parent: %s", bh.Miner, bh.Cid(), parent)
+				return parent, true, nil
 			}
 		}
 	}
 
 	if err := f.byParents.Put(ctx, parentsKey, bh.Cid().Bytes()); err != nil {
-		return cid.Undef, xerrors.Errorf("putting byEpoch entry: %w", err)
+		return cid.Undef, false, xerrors.Errorf("putting byEpoch entry: %w", err)
 	}
 
 	if err := f.byEpoch.Put(ctx, epochKey, bh.Cid().Bytes()); err != nil {
-		return cid.Undef, xerrors.Errorf("putting byEpoch entry: %w", err)
+		return cid.Undef, false, xerrors.Errorf("putting byEpoch entry: %w", err)
 	}
 
-	return cid.Undef, nil
+	return cid.Undef, false, nil
 }
 
-func checkFault(ctx context.Context, t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType string) (cid.Cid, error) {
+func checkFault(ctx context.Context, t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType string) (cid.Cid, bool, error) {
 	fault, err := t.Has(ctx, key)
 	if err != nil {
-		return cid.Undef, xerrors.Errorf("failed to read from datastore: %w", err)
+		return cid.Undef, false, xerrors.Errorf("failed to read from datastore: %w", err)
 	}
 
 	if fault {
 		cidb, err := t.Get(ctx, key)
 		if err != nil {
-			return cid.Undef, xerrors.Errorf("getting other block cid: %w", err)
+			return cid.Undef, false, xerrors.Errorf("getting other block cid: %w", err)
 		}
 
 		_, other, err := cid.CidFromBytes(cidb)
 		if err != nil {
-			return cid.Undef, err
+			return cid.Undef, false, xerrors.Errorf("failed to read cid of other block: %w", err)
 		}
 
 		if other == bh.Cid() {
-			return cid.Undef, nil
+			return cid.Undef, false, nil
 		}
 
-		return other, xerrors.Errorf("produced block would trigger '%s' consensus fault; miner: %s; bh: %s, other: %s", faultType, bh.Miner, bh.Cid(), other)
+		return other, true, nil
 	}
 
-	return cid.Undef, nil
+	return cid.Undef, false, nil
 }

cmd/lotus/daemon.go

@@ -16,7 +16,6 @@ import (
 	"strings"
 
 	"github.com/DataDog/zstd"
-	"github.com/ipfs/go-cid"
 	levelds "github.com/ipfs/go-ds-leveldb"
 	metricsprom "github.com/ipfs/go-metrics-prometheus"
 	"github.com/mitchellh/go-homedir"
@@ -406,7 +405,7 @@ var DaemonCmd = &cli.Command{
 			go func() {
 				err := slashConsensus(api, cctx.String("slashdb-dir"), cctx.String("slasher-sender"))
 				if err != nil {
-					panic("slashConsensus error")
+					panic("slashConsensus error: " + err.Error())
 				}
 			}()
 		}
@@ -639,11 +638,13 @@ func slashConsensus(a lapi.FullNode, p string, from string) error {
 	}
 	for block := range blocks {
 		log.Infof("deal with block: %d, %v, %s", block.Height, block.Miner, block.Cid())
-		if otherBlock, extraBlock, err := slashFilterMinedBlock(ctx, sf, a, block); err != nil {
+		otherBlock, extraBlock, fault, err := slashFilterMinedBlock(ctx, sf, a, block)
-			if otherBlock == nil {
+		if err != nil {
-				continue
+			log.Errorf("slash detector errored: %s", err)
-			}
+			continue
-			log.Errorf("<!!> SLASH FILTER ERROR: %s", err)
+		}
+		if fault {
+			log.Errorf("<!!> SLASH FILTER DETECTED FAULT DUE TO BLOCKS %s and %s", otherBlock.Cid(), block.Cid())
 			bh1, err := cborutil.Dump(otherBlock)
 			if err != nil {
 				log.Errorf("could not dump otherblock:%s, err:%s", otherBlock.Cid(), err)
@@ -682,7 +683,7 @@ func slashConsensus(a lapi.FullNode, p string, from string) error {
 				Params: enc,
 			}, nil)
 			if err != nil {
-				log.Errorf("ReportConsensusFault to messagepool error:%w", err)
+				log.Errorf("ReportConsensusFault to messagepool error:%s", err)
 				continue
 			}
 			log.Infof("ReportConsensusFault message CID:%s", message.Cid())
@@ -692,20 +693,35 @@ func slashConsensus(a lapi.FullNode, p string, from string) error {
 	return err
 }
 
-func slashFilterMinedBlock(ctx context.Context, sf *slashfilter.SlashFilter, a lapi.FullNode, blockB *types.BlockHeader) (*types.BlockHeader, *types.BlockHeader, error) {
+func slashFilterMinedBlock(ctx context.Context, sf *slashfilter.SlashFilter, a lapi.FullNode, blockB *types.BlockHeader) (*types.BlockHeader, *types.BlockHeader, bool, error) {
 	blockC, err := a.ChainGetBlock(ctx, blockB.Parents[0])
 	if err != nil {
-		return nil, nil, xerrors.Errorf("chain get block error:%s", err)
+		return nil, nil, false, xerrors.Errorf("chain get block error:%s", err)
 	}
-	otherCid, err := sf.MinedBlock(ctx, blockB, blockC.Height)
+
+	blockACid, fault, err := sf.MinedBlock(ctx, blockB, blockC.Height)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("slash filter check block error:%s", err)
+		return nil, nil, false, xerrors.Errorf("slash filter check block error:%s", err)
 	}
-	if otherCid != cid.Undef {
+
-		otherHeader, err := a.ChainGetBlock(ctx, otherCid)
+	if !fault {
-		return otherHeader, nil, xerrors.Errorf("chain get other block error:%s", err)
+		return nil, nil, false, nil
+	}
+
+	blockA, err := a.ChainGetBlock(ctx, blockACid)
+	if err != nil {
+		return nil, nil, false, xerrors.Errorf("failed to get blockA: %w", err)
+	}
+
+	// (a) double-fork mining (2 blocks at one epoch)
+	if blockA.Height == blockB.Height {
+		return blockA, nil, true, nil
+	}
+
+	// (b) time-offset mining faults (2 blocks with the same parents)
+	if types.CidArrsEqual(blockB.Parents, blockA.Parents) {
+		return blockA, nil, true, nil
 	}
-	blockA, err := a.ChainGetBlock(ctx, otherCid)
 
 	// (c) parent-grinding fault
 	// Here extra is the "witness", a third block that shows the connection between A and B as
@@ -717,8 +733,9 @@ func slashFilterMinedBlock(ctx context.Context, sf *slashfilter.SlashFilter, a l
 	//  [A, C]
 	if types.CidArrsEqual(blockA.Parents, blockC.Parents) && blockA.Height == blockC.Height &&
 		types.CidArrsContains(blockB.Parents, blockC.Cid()) && !types.CidArrsContains(blockB.Parents, blockA.Cid()) {
-		return blockA, blockC, xerrors.Errorf("chain get other block error:%s", err)
+		return blockA, blockC, true, nil
 	}
 
-	return nil, nil, nil
+	log.Error("unexpectedly reached end of slashFilterMinedBlock despite fault being reported!")
+	return nil, nil, false, nil
 }

miner/miner.go

@@ -324,9 +324,16 @@ minerLoop:
 					"block-time", btime, "time", build.Clock.Now(), "difference", build.Clock.Since(btime))
 			}
 
-			if _, err = m.sf.MinedBlock(ctx, b.Header, base.TipSet.Height()+base.NullRounds); err != nil {
+			if os.Getenv("LOTUS_MINER_NO_SLASHFILTER") != "_yes_i_know_i_can_and_probably_will_lose_all_my_fil_and_power_" {
-				log.Errorf("<!!> SLASH FILTER ERROR: %s", err)
+				witness, fault, err := m.sf.MinedBlock(ctx, b.Header, base.TipSet.Height()+base.NullRounds)
-				if os.Getenv("LOTUS_MINER_NO_SLASHFILTER") != "_yes_i_know_i_can_and_probably_will_lose_all_my_fil_and_power_" {
+				if err != nil {
+					log.Errorf("<!!> SLASH FILTER ERRORED: %s", err)
+					// Continue here, because it's _probably_ wiser to not submit this block
+					continue
+				}
+
+				if fault {
+					log.Errorf("<!!> SLASH FILTER DETECTED FAULT due to blocks %s and %s", b.Header.Cid(), witness)
 					continue
 				}
 			}

node/impl/full/sync.go

@@ -58,9 +58,16 @@ func (a *SyncAPI) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) erro
 	}
 
 	if a.SlashFilter != nil && os.Getenv("LOTUS_NO_SLASHFILTER") != "_yes_i_know_i_can_and_probably_will_lose_all_my_fil_and_power_" {
-		if _, err = a.SlashFilter.MinedBlock(ctx, blk.Header, parent.Height); err != nil {
+		witness, fault, err := a.SlashFilter.MinedBlock(ctx, blk.Header, parent.Height)
-			log.Errorf("<!!> SLASH FILTER ERROR: %s", err)
+		if err != nil {
-			return xerrors.Errorf("<!!> SLASH FILTER ERROR: %w", err)
+			log.Errorf("<!!> SLASH FILTER ERRORED: %s", err)
+			// Return an error here, because it's _probably_ wiser to not submit this block
+			return xerrors.Errorf("<!!> SLASH FILTER ERRORED: %w", err)
+		}
+
+		if fault {
+			log.Errorf("<!!> SLASH FILTER DETECTED FAULT due to witness %s", witness)
+			return xerrors.Errorf("<!!> SLASH FILTER DETECTED FAULT due to witness %s", witness)
 		}
 	}