From 4cfdf8e83c2448ab34dda634a9899bdbde21c37f Mon Sep 17 00:00:00 2001 From: Aayush Date: Tue, 13 Jun 2023 10:21:48 -0400 Subject: [PATCH 1/3] feat: slasher: print error on failure --- cmd/lotus/daemon.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 106446c0a..44082e058 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -406,7 +406,7 @@ var DaemonCmd = &cli.Command{ go func() { err := slashConsensus(api, cctx.String("slashdb-dir"), cctx.String("slasher-sender")) if err != nil { - panic("slashConsensus error") + panic("slashConsensus error: " + err.Error()) } }() } @@ -697,15 +697,19 @@ func slashFilterMinedBlock(ctx context.Context, sf *slashfilter.SlashFilter, a l if err != nil { return nil, nil, xerrors.Errorf("chain get block error:%s", err) } + otherCid, err := sf.MinedBlock(ctx, blockB, blockC.Height) if err != nil { return nil, nil, xerrors.Errorf("slash filter check block error:%s", err) } + if otherCid != cid.Undef { otherHeader, err := a.ChainGetBlock(ctx, otherCid) return otherHeader, nil, xerrors.Errorf("chain get other block error:%s", err) } + blockA, err := a.ChainGetBlock(ctx, otherCid) + if // (c) parent-grinding fault // Here extra is the "witness", a third block that shows the connection between A and B as From 802a9f0a780411c76e9b25ed076798e1414fbf74 Mon Sep 17 00:00:00 2001 From: Aayush Date: Tue, 13 Jun 2023 10:35:02 -0400 Subject: [PATCH 2/3] feat: refactor slashfilter to return bool indicating fault --- chain/gen/slashfilter/slashfilter.go | 48 +++++++++++++++++----------- miner/miner.go | 13 ++++++-- node/impl/full/sync.go | 13 ++++++-- 3 files changed, 49 insertions(+), 25 deletions(-) diff --git a/chain/gen/slashfilter/slashfilter.go b/chain/gen/slashfilter/slashfilter.go index 0e6b00cfb..71b5dad9a 100644 --- a/chain/gen/slashfilter/slashfilter.go +++ b/chain/gen/slashfilter/slashfilter.go @@ -26,20 +26,30 @@ func New(dstore ds.Batching) *SlashFilter { } } -func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, parentEpoch abi.ChainEpoch) (cid.Cid, error) { +func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, parentEpoch abi.ChainEpoch) (cid.Cid, bool, error) { epochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, bh.Height)) { // double-fork mining (2 blocks at one epoch) - if witness, err := checkFault(ctx, f.byEpoch, epochKey, bh, "double-fork mining faults"); err != nil { - return witness, xerrors.Errorf("check double-fork mining faults: %w", err) + doubleForkWitness, doubleForkFault, err := checkFault(ctx, f.byEpoch, epochKey, bh, "double-fork mining faults") + if err != nil { + return cid.Undef, false, xerrors.Errorf("check double-fork mining faults: %w", err) + } + + if doubleForkFault { + return doubleForkWitness, doubleForkFault, nil } } parentsKey := ds.NewKey(fmt.Sprintf("/%s/%x", bh.Miner, types.NewTipSetKey(bh.Parents...).Bytes())) { // time-offset mining faults (2 blocks with the same parents) - if witness, err := checkFault(ctx, f.byParents, parentsKey, bh, "time-offset mining faults"); err != nil { - return witness, xerrors.Errorf("check time-offset mining faults: %w", err) + timeOffsetWitness, timeOffsetFault, err := checkFault(ctx, f.byParents, parentsKey, bh, "time-offset mining faults") + if err != nil { + return cid.Undef, false, xerrors.Errorf("check time-offset mining faults: %w", err) + } + + if timeOffsetFault { + return timeOffsetWitness, timeOffsetFault, nil } } @@ -50,19 +60,19 @@ func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, par parentEpochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, parentEpoch)) have, err := f.byEpoch.Has(ctx, parentEpochKey) if err != nil { - return cid.Undef, err + return cid.Undef, false, xerrors.Errorf("failed to read from db: %w", err) } if have { // If we had, make sure it's in our parent tipset cidb, err := f.byEpoch.Get(ctx, parentEpochKey) if err != nil { - return cid.Undef, xerrors.Errorf("getting other block cid: %w", err) + return cid.Undef, false, xerrors.Errorf("getting other block cid: %w", err) } _, parent, err := cid.CidFromBytes(cidb) if err != nil { - return cid.Undef, err + return cid.Undef, false, xerrors.Errorf("failed to read cid from bytes: %w", err) } var found bool @@ -73,45 +83,45 @@ func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, par } if !found { - return parent, xerrors.Errorf("produced block would trigger 'parent-grinding fault' consensus fault; miner: %s; bh: %s, expected parent: %s", bh.Miner, bh.Cid(), parent) + return parent, true, nil } } } if err := f.byParents.Put(ctx, parentsKey, bh.Cid().Bytes()); err != nil { - return cid.Undef, xerrors.Errorf("putting byEpoch entry: %w", err) + return cid.Undef, false, xerrors.Errorf("putting byEpoch entry: %w", err) } if err := f.byEpoch.Put(ctx, epochKey, bh.Cid().Bytes()); err != nil { - return cid.Undef, xerrors.Errorf("putting byEpoch entry: %w", err) + return cid.Undef, false, xerrors.Errorf("putting byEpoch entry: %w", err) } - return cid.Undef, nil + return cid.Undef, false, nil } -func checkFault(ctx context.Context, t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType string) (cid.Cid, error) { +func checkFault(ctx context.Context, t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType string) (cid.Cid, bool, error) { fault, err := t.Has(ctx, key) if err != nil { - return cid.Undef, xerrors.Errorf("failed to read from datastore: %w", err) + return cid.Undef, false, xerrors.Errorf("failed to read from datastore: %w", err) } if fault { cidb, err := t.Get(ctx, key) if err != nil { - return cid.Undef, xerrors.Errorf("getting other block cid: %w", err) + return cid.Undef, false, xerrors.Errorf("getting other block cid: %w", err) } _, other, err := cid.CidFromBytes(cidb) if err != nil { - return cid.Undef, err + return cid.Undef, false, xerrors.Errorf("failed to read cid of other block: %w", err) } if other == bh.Cid() { - return cid.Undef, nil + return cid.Undef, false, nil } - return other, xerrors.Errorf("produced block would trigger '%s' consensus fault; miner: %s; bh: %s, other: %s", faultType, bh.Miner, bh.Cid(), other) + return other, true, nil } - return cid.Undef, nil + return cid.Undef, false, nil } diff --git a/miner/miner.go b/miner/miner.go index e1737009b..1caaade96 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -324,9 +324,16 @@ minerLoop: "block-time", btime, "time", build.Clock.Now(), "difference", build.Clock.Since(btime)) } - if _, err = m.sf.MinedBlock(ctx, b.Header, base.TipSet.Height()+base.NullRounds); err != nil { - log.Errorf(" SLASH FILTER ERROR: %s", err) - if os.Getenv("LOTUS_MINER_NO_SLASHFILTER") != "_yes_i_know_i_can_and_probably_will_lose_all_my_fil_and_power_" { + if os.Getenv("LOTUS_MINER_NO_SLASHFILTER") != "_yes_i_know_i_can_and_probably_will_lose_all_my_fil_and_power_" { + witness, fault, err := m.sf.MinedBlock(ctx, b.Header, base.TipSet.Height()+base.NullRounds) + if err != nil { + log.Errorf(" SLASH FILTER ERRORED: %s", err) + // Continue here, because it's _probably_ wiser to not submit this block + continue + } + + if fault { + log.Errorf(" SLASH FILTER DETECTED FAULT due to witness %s", witness) continue } } diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index 9be43338e..223f5c29e 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -58,9 +58,16 @@ func (a *SyncAPI) SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) erro } if a.SlashFilter != nil && os.Getenv("LOTUS_NO_SLASHFILTER") != "_yes_i_know_i_can_and_probably_will_lose_all_my_fil_and_power_" { - if _, err = a.SlashFilter.MinedBlock(ctx, blk.Header, parent.Height); err != nil { - log.Errorf(" SLASH FILTER ERROR: %s", err) - return xerrors.Errorf(" SLASH FILTER ERROR: %w", err) + witness, fault, err := a.SlashFilter.MinedBlock(ctx, blk.Header, parent.Height) + if err != nil { + log.Errorf(" SLASH FILTER ERRORED: %s", err) + // Return an error here, because it's _probably_ wiser to not submit this block + return xerrors.Errorf(" SLASH FILTER ERRORED: %w", err) + } + + if fault { + log.Errorf(" SLASH FILTER DETECTED FAULT due to witness %s", witness) + return xerrors.Errorf(" SLASH FILTER DETECTED FAULT due to witness %s", witness) } } From 7180b52d2cf7ae92ed2145ffb1a9c9670aa199e7 Mon Sep 17 00:00:00 2001 From: Aayush Date: Tue, 13 Jun 2023 10:51:03 -0400 Subject: [PATCH 3/3] feat: improvements to the consensus slasher --- cmd/lotus/daemon.go | 49 ++++++++++++++++++++++++++++----------------- miner/miner.go | 2 +- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 44082e058..9fa1d42ae 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -16,7 +16,6 @@ import ( "strings" "github.com/DataDog/zstd" - "github.com/ipfs/go-cid" levelds "github.com/ipfs/go-ds-leveldb" metricsprom "github.com/ipfs/go-metrics-prometheus" "github.com/mitchellh/go-homedir" @@ -639,11 +638,13 @@ func slashConsensus(a lapi.FullNode, p string, from string) error { } for block := range blocks { log.Infof("deal with block: %d, %v, %s", block.Height, block.Miner, block.Cid()) - if otherBlock, extraBlock, err := slashFilterMinedBlock(ctx, sf, a, block); err != nil { - if otherBlock == nil { - continue - } - log.Errorf(" SLASH FILTER ERROR: %s", err) + otherBlock, extraBlock, fault, err := slashFilterMinedBlock(ctx, sf, a, block) + if err != nil { + log.Errorf("slash detector errored: %s", err) + continue + } + if fault { + log.Errorf(" SLASH FILTER DETECTED FAULT DUE TO BLOCKS %s and %s", otherBlock.Cid(), block.Cid()) bh1, err := cborutil.Dump(otherBlock) if err != nil { log.Errorf("could not dump otherblock:%s, err:%s", otherBlock.Cid(), err) @@ -682,7 +683,7 @@ func slashConsensus(a lapi.FullNode, p string, from string) error { Params: enc, }, nil) if err != nil { - log.Errorf("ReportConsensusFault to messagepool error:%w", err) + log.Errorf("ReportConsensusFault to messagepool error:%s", err) continue } log.Infof("ReportConsensusFault message CID:%s", message.Cid()) @@ -692,24 +693,35 @@ func slashConsensus(a lapi.FullNode, p string, from string) error { return err } -func slashFilterMinedBlock(ctx context.Context, sf *slashfilter.SlashFilter, a lapi.FullNode, blockB *types.BlockHeader) (*types.BlockHeader, *types.BlockHeader, error) { +func slashFilterMinedBlock(ctx context.Context, sf *slashfilter.SlashFilter, a lapi.FullNode, blockB *types.BlockHeader) (*types.BlockHeader, *types.BlockHeader, bool, error) { blockC, err := a.ChainGetBlock(ctx, blockB.Parents[0]) if err != nil { - return nil, nil, xerrors.Errorf("chain get block error:%s", err) + return nil, nil, false, xerrors.Errorf("chain get block error:%s", err) } - otherCid, err := sf.MinedBlock(ctx, blockB, blockC.Height) + blockACid, fault, err := sf.MinedBlock(ctx, blockB, blockC.Height) if err != nil { - return nil, nil, xerrors.Errorf("slash filter check block error:%s", err) + return nil, nil, false, xerrors.Errorf("slash filter check block error:%s", err) } - if otherCid != cid.Undef { - otherHeader, err := a.ChainGetBlock(ctx, otherCid) - return otherHeader, nil, xerrors.Errorf("chain get other block error:%s", err) + if !fault { + return nil, nil, false, nil } - blockA, err := a.ChainGetBlock(ctx, otherCid) - if + blockA, err := a.ChainGetBlock(ctx, blockACid) + if err != nil { + return nil, nil, false, xerrors.Errorf("failed to get blockA: %w", err) + } + + // (a) double-fork mining (2 blocks at one epoch) + if blockA.Height == blockB.Height { + return blockA, nil, true, nil + } + + // (b) time-offset mining faults (2 blocks with the same parents) + if types.CidArrsEqual(blockB.Parents, blockA.Parents) { + return blockA, nil, true, nil + } // (c) parent-grinding fault // Here extra is the "witness", a third block that shows the connection between A and B as @@ -721,8 +733,9 @@ func slashFilterMinedBlock(ctx context.Context, sf *slashfilter.SlashFilter, a l // [A, C] if types.CidArrsEqual(blockA.Parents, blockC.Parents) && blockA.Height == blockC.Height && types.CidArrsContains(blockB.Parents, blockC.Cid()) && !types.CidArrsContains(blockB.Parents, blockA.Cid()) { - return blockA, blockC, xerrors.Errorf("chain get other block error:%s", err) + return blockA, blockC, true, nil } - return nil, nil, nil + log.Error("unexpectedly reached end of slashFilterMinedBlock despite fault being reported!") + return nil, nil, false, nil } diff --git a/miner/miner.go b/miner/miner.go index 1caaade96..9281854d7 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -333,7 +333,7 @@ minerLoop: } if fault { - log.Errorf(" SLASH FILTER DETECTED FAULT due to witness %s", witness) + log.Errorf(" SLASH FILTER DETECTED FAULT due to blocks %s and %s", b.Header.Cid(), witness) continue } }