From 14754f1b1875a1477cd7153cc5cffc0665b6874d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 2 Aug 2021 14:08:44 -0700 Subject: [PATCH 1/9] chore: dedup datastore import --- chain/store/store.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index df5936c37..a8e378e64 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -31,7 +31,6 @@ import ( lru "github.com/hashicorp/golang-lru" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" dstore "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" cbor "github.com/ipfs/go-ipld-cbor" @@ -642,7 +641,7 @@ func (cs *ChainStore) FlushValidationCache() error { return FlushValidationCache(cs.metadataDs) } -func FlushValidationCache(ds datastore.Batching) error { +func FlushValidationCache(ds dstore.Batching) error { log.Infof("clearing block validation cache...") dsWalk, err := ds.Query(query.Query{ @@ -674,7 +673,7 @@ func FlushValidationCache(ds datastore.Batching) error { for _, k := range allKeys { if strings.HasPrefix(k.Key, blockValidationCacheKeyPrefix.String()) { delCnt++ - batch.Delete(datastore.RawKey(k.Key)) // nolint:errcheck + batch.Delete(dstore.RawKey(k.Key)) // nolint:errcheck } } From 43bbde1e6b922b7ac497bf9c2c7330d2681d1cba Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 2 Aug 2021 14:12:00 -0700 Subject: [PATCH 2/9] fix: close chain head subscription when the reader is slow The reader can just re-subscribe when they're ready to catch up. This prevents a slow reader from bogging down the entire system. --- chain/store/store.go | 29 +++++++++++++++++---------- itests/api_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++++ tools/stats/rpc.go | 5 ++++- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index a8e378e64..1c90b7e0c 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -293,27 +293,36 @@ func (cs *ChainStore) SubHeadChanges(ctx context.Context) chan []*api.HeadChange }} go func() { - defer close(out) - var unsubOnce sync.Once + defer func() { + // Tell the caller we're done first, the following may block for a bit. + close(out) + + // Unsubscribe. + cs.bestTips.Unsub(subch) + + // Drain the channel. + for range subch { + } + }() for { select { case val, ok := <-subch: if !ok { - log.Warn("chain head sub exit loop") + // Shutting down. + return + } + select { + case out <- val.([]*api.HeadChange): + default: + log.Errorf("closing head change subscription due to slow reader") return } if len(out) > 5 { log.Warnf("head change sub is slow, has %d buffered entries", len(out)) } - select { - case out <- val.([]*api.HeadChange): - case <-ctx.Done(): - } case <-ctx.Done(): - unsubOnce.Do(func() { - go cs.bestTips.Unsub(subch) - }) + return } } }() diff --git a/itests/api_test.go b/itests/api_test.go index 01e006fed..9a21c9dfc 100644 --- a/itests/api_test.go +++ b/itests/api_test.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/itests/kit" + logging "github.com/ipfs/go-log/v2" ) func TestAPI(t *testing.T) { @@ -39,6 +40,7 @@ func runAPITest(t *testing.T, opts ...interface{}) { t.Run("testConnectTwo", ts.testConnectTwo) t.Run("testMining", ts.testMining) t.Run("testMiningReal", ts.testMiningReal) + t.Run("testSlowNotify", ts.testSlowNotify) t.Run("testSearchMsg", ts.testSearchMsg) t.Run("testNonGenesisMiner", ts.testNonGenesisMiner) } @@ -169,6 +171,51 @@ func (ts *apiSuite) testMiningReal(t *testing.T) { ts.testMining(t) } +func (ts *apiSuite) testSlowNotify(t *testing.T) { + _ = logging.SetLogLevel("rpc", "ERROR") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + full, miner, _ := kit.EnsembleMinimal(t, ts.opts...) + + // Subscribe a bunch of times to make sure we fill up any RPC buffers. + var newHeadsChans []<-chan []*lapi.HeadChange + for i := 0; i < 100; i++ { + newHeads, err := full.ChainNotify(ctx) + require.NoError(t, err) + newHeadsChans = append(newHeadsChans, newHeads) + } + + initHead := (<-newHeadsChans[0])[0] + baseHeight := initHead.Val.Height() + + bm := kit.NewBlockMiner(t, miner) + bm.MineBlocks(ctx, time.Microsecond) + + full.WaitTillChain(ctx, kit.HeightAtLeast(baseHeight+100)) + + // Make sure they were all closed. + for _, ch := range newHeadsChans { + var ok bool + for ok { + select { + case _, ok = <-ch: + default: + t.Fatal("expected new heads channel to be closed") + } + } + } + + // Make sure we can resubscribe and everything still works. + newHeads, err := full.ChainNotify(ctx) + require.NoError(t, err) + for i := 0; i < 10; i++ { + _, ok := <-newHeads + require.True(t, ok, "notify channel closed") + } +} + func (ts *apiSuite) testNonGenesisMiner(t *testing.T) { ctx := context.Background() diff --git a/tools/stats/rpc.go b/tools/stats/rpc.go index 0aa3d141e..4e503cb39 100644 --- a/tools/stats/rpc.go +++ b/tools/stats/rpc.go @@ -139,7 +139,10 @@ func GetTips(ctx context.Context, api v0api.FullNode, lastHeight abi.ChainEpoch, for { select { - case changes := <-notif: + case changes, ok := <-notif: + if !ok { + return + } for _, change := range changes { log.Infow("Head event", "height", change.Val.Height(), "type", change.Type) From a875e9ba732bffc05fb6890ca47a942871f0107c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 3 Aug 2021 16:30:14 -0700 Subject: [PATCH 3/9] fix: check parents when adding tipsets to the "cache" --- chain/events/tscache.go | 11 +- chain/events/tscache_test.go | 203 +++++++++++++++++++---------------- 2 files changed, 121 insertions(+), 93 deletions(-) diff --git a/chain/events/tscache.go b/chain/events/tscache.go index 44699684e..7beb80364 100644 --- a/chain/events/tscache.go +++ b/chain/events/tscache.go @@ -21,7 +21,7 @@ type tipSetCache struct { mu sync.RWMutex cache []*types.TipSet - start int + start int // chain head (end) len int storage tsCacheAPI @@ -42,9 +42,16 @@ func (tsc *tipSetCache) add(ts *types.TipSet) error { defer tsc.mu.Unlock() if tsc.len > 0 { - if tsc.cache[tsc.start].Height() >= ts.Height() { + best := tsc.cache[tsc.start] + if best.Height() >= ts.Height() { return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.cache[tsc.start].Height()+1, ts.Height()) } + if best.Key() != ts.Parents() { + return xerrors.Errorf( + "tipSetCache.add: expected new tipset %s (%d) to follow %s (%d), its parents are %s", + ts.Key(), ts.Height(), best.Key(), best.Height(), best.Parents(), + ) + } } nextH := ts.Height() diff --git a/chain/events/tscache_test.go b/chain/events/tscache_test.go index ab6336f24..9ba9a556c 100644 --- a/chain/events/tscache_test.go +++ b/chain/events/tscache_test.go @@ -6,57 +6,13 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" ) -func TestTsCache(t *testing.T) { - tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t}) - - h := abi.ChainEpoch(75) - - a, _ := address.NewFromString("t00") - - add := func() { - ts, err := types.NewTipSet([]*types.BlockHeader{{ - Miner: a, - Height: h, - ParentStateRoot: dummyCid, - Messages: dummyCid, - ParentMessageReceipts: dummyCid, - BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, - BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, - }}) - if err != nil { - t.Fatal(err) - } - if err := tsc.add(ts); err != nil { - t.Fatal(err) - } - h++ - } - - for i := 0; i < 9000; i++ { - if i%90 > 60 { - best, err := tsc.best() - if err != nil { - t.Fatal(err, "; i:", i) - return - } - if err := tsc.revert(best); err != nil { - t.Fatal(err, "; i:", i) - return - } - h-- - } else { - add() - } - } - -} - type tsCacheAPIFailOnStorageCall struct { t *testing.T } @@ -70,77 +26,123 @@ func (tc *tsCacheAPIFailOnStorageCall) ChainHead(ctx context.Context) (*types.Ti return &types.TipSet{}, nil } -func TestTsCacheNulls(t *testing.T) { - tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t}) +type cacheHarness struct { + t *testing.T - h := abi.ChainEpoch(75) + miner address.Address + tsc *tipSetCache + height abi.ChainEpoch +} - a, _ := address.NewFromString("t00") - add := func() { - ts, err := types.NewTipSet([]*types.BlockHeader{{ - Miner: a, - Height: h, - ParentStateRoot: dummyCid, - Messages: dummyCid, - ParentMessageReceipts: dummyCid, - BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, - BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, - }}) - if err != nil { - t.Fatal(err) - } - if err := tsc.add(ts); err != nil { - t.Fatal(err) - } - h++ +func newCacheharness(t *testing.T) *cacheHarness { + a, err := address.NewFromString("t00") + require.NoError(t, err) + + h := &cacheHarness{ + t: t, + tsc: newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t}), + height: 75, + miner: a, } + h.addWithParents(nil) + return h +} - add() - add() - add() - h += 5 +func (h *cacheHarness) addWithParents(parents []cid.Cid) { + ts, err := types.NewTipSet([]*types.BlockHeader{{ + Miner: h.miner, + Height: h.height, + ParentStateRoot: dummyCid, + Messages: dummyCid, + ParentMessageReceipts: dummyCid, + BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, + BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, + Parents: parents, + }}) + require.NoError(h.t, err) + require.NoError(h.t, h.tsc.add(ts)) + h.height++ +} - add() - add() +func (h *cacheHarness) add() { + last, err := h.tsc.best() + require.NoError(h.t, err) + h.addWithParents(last.Cids()) +} - best, err := tsc.best() +func (h *cacheHarness) revert() { + best, err := h.tsc.best() + require.NoError(h.t, err) + err = h.tsc.revert(best) + require.NoError(h.t, err) + h.height-- +} + +func (h *cacheHarness) skip(n abi.ChainEpoch) { + h.height += n +} + +func TestTsCache(t *testing.T) { + h := newCacheharness(t) + + for i := 0; i < 9000; i++ { + if i%90 > 60 { + h.revert() + } else { + h.add() + } + } +} + +func TestTsCacheNulls(t *testing.T) { + h := newCacheharness(t) + + h.add() + h.add() + h.add() + h.skip(5) + + h.add() + h.add() + + best, err := h.tsc.best() require.NoError(t, err) - require.Equal(t, h-1, best.Height()) + require.Equal(t, h.height-1, best.Height()) - ts, err := tsc.get(h - 1) + ts, err := h.tsc.get(h.height - 1) require.NoError(t, err) - require.Equal(t, h-1, ts.Height()) + require.Equal(t, h.height-1, ts.Height()) - ts, err = tsc.get(h - 2) + ts, err = h.tsc.get(h.height - 2) require.NoError(t, err) - require.Equal(t, h-2, ts.Height()) + require.Equal(t, h.height-2, ts.Height()) - ts, err = tsc.get(h - 3) + ts, err = h.tsc.get(h.height - 3) require.NoError(t, err) require.Nil(t, ts) - ts, err = tsc.get(h - 8) + ts, err = h.tsc.get(h.height - 8) require.NoError(t, err) - require.Equal(t, h-8, ts.Height()) + require.Equal(t, h.height-8, ts.Height()) - best, err = tsc.best() + best, err = h.tsc.best() require.NoError(t, err) - require.NoError(t, tsc.revert(best)) + require.NoError(t, h.tsc.revert(best)) - best, err = tsc.best() + best, err = h.tsc.best() require.NoError(t, err) - require.NoError(t, tsc.revert(best)) + require.NoError(t, h.tsc.revert(best)) - best, err = tsc.best() + best, err = h.tsc.best() require.NoError(t, err) - require.Equal(t, h-8, best.Height()) + require.Equal(t, h.height-8, best.Height()) - h += 50 - add() + h.skip(50) + h.add() - ts, err = tsc.get(h - 1) + ts, err = h.tsc.get(h.height - 1) require.NoError(t, err) - require.Equal(t, h-1, ts.Height()) + require.Equal(t, h.height-1, ts.Height()) } type tsCacheAPIStorageCallCounter struct { @@ -166,3 +168,22 @@ func TestTsCacheEmpty(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, callCounter.chainHead) } + +func TestTsCacheSkip(t *testing.T) { + h := newCacheharness(t) + + ts, err := types.NewTipSet([]*types.BlockHeader{{ + Miner: h.miner, + Height: h.height, + ParentStateRoot: dummyCid, + Messages: dummyCid, + ParentMessageReceipts: dummyCid, + BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, + BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, + // With parents that don't match the last block. + Parents: nil, + }}) + require.NoError(h.t, err) + err = h.tsc.add(ts) + require.Error(t, err) +} From 38461703028c86c8b7f8a55950cf117c82e0026c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 3 Aug 2021 17:10:30 -0700 Subject: [PATCH 4/9] refactor events system --- chain/events/cache.go | 33 ++ chain/events/events.go | 205 +-------- chain/events/events_called.go | 275 +++++------- chain/events/events_height.go | 395 +++++++++-------- chain/events/events_test.go | 401 +++++++----------- chain/events/message_cache.go | 42 ++ chain/events/observer.go | 234 ++++++++++ chain/events/tscache.go | 207 +++++---- chain/events/tscache_test.go | 61 ++- chain/events/utils.go | 6 +- itests/paych_api_test.go | 5 +- itests/paych_cli_test.go | 5 +- markets/storageadapter/client.go | 11 +- .../storageadapter/ondealsectorcommitted.go | 10 +- .../ondealsectorcommitted_test.go | 6 +- markets/storageadapter/provider.go | 13 +- paychmgr/settler/settler.go | 9 +- storage/adapter_events.go | 2 +- storage/miner.go | 38 +- 19 files changed, 1041 insertions(+), 917 deletions(-) create mode 100644 chain/events/cache.go create mode 100644 chain/events/message_cache.go create mode 100644 chain/events/observer.go diff --git a/chain/events/cache.go b/chain/events/cache.go new file mode 100644 index 000000000..ef4b5bba8 --- /dev/null +++ b/chain/events/cache.go @@ -0,0 +1,33 @@ +package events + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" +) + +type uncachedAPI interface { + ChainNotify(context.Context) (<-chan []*api.HeadChange, error) + ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) + StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) + + StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // optional / for CalledMsg +} + +type cache struct { + *tipSetCache + *messageCache + uncachedAPI +} + +func newCache(api EventAPI, gcConfidence abi.ChainEpoch) *cache { + return &cache{ + newTSCache(api, gcConfidence), + newMessageCache(api), + api, + } +} diff --git a/chain/events/events.go b/chain/events/events.go index 8511de921..1e39d3646 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -2,18 +2,14 @@ package events import ( "context" - "sync" - "time" "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) @@ -25,209 +21,50 @@ type ( RevertHandler func(ctx context.Context, ts *types.TipSet) error ) -type heightHandler struct { - confidence int - called bool - - handle HeightHandler - revert RevertHandler +// A TipSetObserver receives notifications of tipsets +type TipSetObserver interface { + Apply(ctx context.Context, from, to *types.TipSet) error + Revert(ctx context.Context, from, to *types.TipSet) error } type EventAPI interface { ChainNotify(context.Context) (<-chan []*api.HeadChange, error) ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) ChainHead(context.Context) (*types.TipSet, error) StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error) + ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // optional / for CalledMsg } type Events struct { - api EventAPI - - tsc *tipSetCache - lk sync.Mutex - - ready chan struct{} - readyOnce sync.Once - - heightEvents + *observer + *heightEvents *hcEvents - - observers []TipSetObserver } -func NewEventsWithConfidence(ctx context.Context, api EventAPI, gcConfidence abi.ChainEpoch) *Events { - tsc := newTSCache(gcConfidence, api) +func NewEventsWithConfidence(ctx context.Context, api EventAPI, gcConfidence abi.ChainEpoch) (*Events, error) { + cache := newCache(api, gcConfidence) - e := &Events{ - api: api, + ob := newObserver(cache, gcConfidence) + he := newHeightEvents(cache, gcConfidence) + headChange := newHCEvents(cache) - tsc: tsc, - - heightEvents: heightEvents{ - tsc: tsc, - ctx: ctx, - gcConfidence: gcConfidence, - - heightTriggers: map[uint64]*heightHandler{}, - htTriggerHeights: map[abi.ChainEpoch][]uint64{}, - htHeights: map[abi.ChainEpoch][]uint64{}, - }, - - hcEvents: newHCEvents(ctx, api, tsc, uint64(gcConfidence)), - ready: make(chan struct{}), - observers: []TipSetObserver{}, + // Cache first. Observers are ordered and we always want to fill the cache first. + ob.Observe(cache.observer()) + ob.Observe(he.observer()) + ob.Observe(headChange.observer()) + if err := ob.start(ctx); err != nil { + return nil, err } - go e.listenHeadChanges(ctx) - - // Wait for the first tipset to be seen or bail if shutting down - select { - case <-e.ready: - case <-ctx.Done(): - } - - return e + return &Events{ob, he, headChange}, nil } -func NewEvents(ctx context.Context, api EventAPI) *Events { +func NewEvents(ctx context.Context, api EventAPI) (*Events, error) { gcConfidence := 2 * build.ForkLengthThreshold return NewEventsWithConfidence(ctx, api, gcConfidence) } - -func (e *Events) listenHeadChanges(ctx context.Context) { - for { - if err := e.listenHeadChangesOnce(ctx); err != nil { - log.Errorf("listen head changes errored: %s", err) - } else { - log.Warn("listenHeadChanges quit") - } - select { - case <-build.Clock.After(time.Second): - case <-ctx.Done(): - log.Warnf("not restarting listenHeadChanges: context error: %s", ctx.Err()) - return - } - - log.Info("restarting listenHeadChanges") - } -} - -func (e *Events) listenHeadChangesOnce(ctx context.Context) error { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - notifs, err := e.api.ChainNotify(ctx) - if err != nil { - // Retry is handled by caller - return xerrors.Errorf("listenHeadChanges ChainNotify call failed: %w", err) - } - - var cur []*api.HeadChange - var ok bool - - // Wait for first tipset or bail - select { - case cur, ok = <-notifs: - if !ok { - return xerrors.Errorf("notification channel closed") - } - case <-ctx.Done(): - return ctx.Err() - } - - if len(cur) != 1 { - return xerrors.Errorf("unexpected initial head notification length: %d", len(cur)) - } - - if cur[0].Type != store.HCCurrent { - return xerrors.Errorf("expected first head notification type to be 'current', was '%s'", cur[0].Type) - } - - if err := e.tsc.add(cur[0].Val); err != nil { - log.Warnf("tsc.add: adding current tipset failed: %v", err) - } - - e.readyOnce.Do(func() { - e.lastTs = cur[0].Val - // Signal that we have seen first tipset - close(e.ready) - }) - - for notif := range notifs { - var rev, app []*types.TipSet - for _, notif := range notif { - switch notif.Type { - case store.HCRevert: - rev = append(rev, notif.Val) - case store.HCApply: - app = append(app, notif.Val) - default: - log.Warnf("unexpected head change notification type: '%s'", notif.Type) - } - } - - if err := e.headChange(ctx, rev, app); err != nil { - log.Warnf("headChange failed: %s", err) - } - - // sync with fake chainstore (for tests) - if fcs, ok := e.api.(interface{ notifDone() }); ok { - fcs.notifDone() - } - } - - return nil -} - -func (e *Events) headChange(ctx context.Context, rev, app []*types.TipSet) error { - if len(app) == 0 { - return xerrors.New("events.headChange expected at least one applied tipset") - } - - e.lk.Lock() - defer e.lk.Unlock() - - if err := e.headChangeAt(rev, app); err != nil { - return err - } - - if err := e.observeChanges(ctx, rev, app); err != nil { - return err - } - return e.processHeadChangeEvent(rev, app) -} - -// A TipSetObserver receives notifications of tipsets -type TipSetObserver interface { - Apply(ctx context.Context, ts *types.TipSet) error - Revert(ctx context.Context, ts *types.TipSet) error -} - -// TODO: add a confidence level so we can have observers with difference levels of confidence -func (e *Events) Observe(obs TipSetObserver) error { - e.lk.Lock() - defer e.lk.Unlock() - e.observers = append(e.observers, obs) - return nil -} - -// observeChanges expects caller to hold e.lk -func (e *Events) observeChanges(ctx context.Context, rev, app []*types.TipSet) error { - for _, ts := range rev { - for _, o := range e.observers { - _ = o.Revert(ctx, ts) - } - } - - for _, ts := range app { - for _, o := range e.observers { - _ = o.Apply(ctx, ts) - } - } - - return nil -} diff --git a/chain/events/events_called.go b/chain/events/events_called.go index 1f0b80169..2b1a76e84 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -5,9 +5,6 @@ import ( "math" "sync" - "github.com/filecoin-project/lotus/api" - lru "github.com/hashicorp/golang-lru" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/go-state-types/abi" @@ -35,7 +32,7 @@ type eventData interface{} // `prevTs` is the previous tipset, eg the "from" tipset for a state change. // `ts` is the event tipset, eg the tipset in which the `msg` is included. // `curH`-`ts.Height` = `confidence` -type EventHandler func(data eventData, prevTs, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error) +type EventHandler func(ctx context.Context, data eventData, prevTs, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error) // CheckFunc is used for atomicity guarantees. If the condition the callbacks // wait for has already happened in tipset `ts` @@ -43,7 +40,7 @@ type EventHandler func(data eventData, prevTs, ts *types.TipSet, curH abi.ChainE // If `done` is true, timeout won't be triggered // If `more` is false, no messages will be sent to EventHandler (RevertHandler // may still be called) -type CheckFunc func(ts *types.TipSet) (done bool, more bool, err error) +type CheckFunc func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) // Keep track of information for an event handler type handlerInfo struct { @@ -60,10 +57,9 @@ type handlerInfo struct { // until the required confidence is reached type queuedEvent struct { trigger triggerID + data eventData - prevH abi.ChainEpoch - h abi.ChainEpoch - data eventData + prevTipset, tipset *types.TipSet called bool } @@ -71,10 +67,7 @@ type queuedEvent struct { // Manages chain head change events, which may be forward (new tipset added to // chain) or backward (chain branch discarded in favour of heavier branch) type hcEvents struct { - cs EventAPI - tsc *tipSetCache - ctx context.Context - gcConfidence uint64 + cs EventAPI lastTs *types.TipSet @@ -82,8 +75,10 @@ type hcEvents struct { ctr triggerID + // TODO: get rid of trigger IDs and just use pointers as keys. triggers map[triggerID]*handlerInfo + // TODO: instead of scheduling events in the future, look at the chain in the past. We can sip the "confidence" queue entirely. // maps block heights to events // [triggerH][msgH][event] confQueue map[triggerH]map[msgH][]*queuedEvent @@ -98,83 +93,76 @@ type hcEvents struct { watcherEvents } -func newHCEvents(ctx context.Context, cs EventAPI, tsc *tipSetCache, gcConfidence uint64) *hcEvents { - e := hcEvents{ - ctx: ctx, - cs: cs, - tsc: tsc, - gcConfidence: gcConfidence, - +func newHCEvents(api EventAPI) *hcEvents { + e := &hcEvents{ + cs: api, confQueue: map[triggerH]map[msgH][]*queuedEvent{}, revertQueue: map[msgH][]triggerH{}, triggers: map[triggerID]*handlerInfo{}, timeouts: map[abi.ChainEpoch]map[triggerID]int{}, } - e.messageEvents = newMessageEvents(ctx, &e, cs) - e.watcherEvents = newWatcherEvents(ctx, &e, cs) + e.messageEvents = newMessageEvents(e, api) + e.watcherEvents = newWatcherEvents(e, api) - return &e + return e } -// Called when there is a change to the head with tipsets to be -// reverted / applied -func (e *hcEvents) processHeadChangeEvent(rev, app []*types.TipSet) error { +type hcEventsObserver hcEvents + +func (e *hcEvents) observer() TipSetObserver { + return (*hcEventsObserver)(e) +} + +func (e *hcEventsObserver) Apply(ctx context.Context, from, to *types.TipSet) error { e.lk.Lock() defer e.lk.Unlock() - for _, ts := range rev { - e.handleReverts(ts) - e.lastTs = ts + defer func() { e.lastTs = to }() + + // Check if the head change caused any state changes that we were + // waiting for + stateChanges := e.checkStateChanges(from, to) + + // Queue up calls until there have been enough blocks to reach + // confidence on the state changes + for tid, data := range stateChanges { + e.queueForConfidence(tid, data, from, to) } - for _, ts := range app { - // Check if the head change caused any state changes that we were - // waiting for - stateChanges := e.watcherEvents.checkStateChanges(e.lastTs, ts) + // Check if the head change included any new message calls + newCalls := e.checkNewCalls(ctx, from, to) - // Queue up calls until there have been enough blocks to reach - // confidence on the state changes - for tid, data := range stateChanges { - e.queueForConfidence(tid, data, e.lastTs, ts) + // Queue up calls until there have been enough blocks to reach + // confidence on the message calls + for tid, calls := range newCalls { + for _, data := range calls { + e.queueForConfidence(tid, data, nil, to) } - - // Check if the head change included any new message calls - newCalls, err := e.messageEvents.checkNewCalls(ts) - if err != nil { - return err - } - - // Queue up calls until there have been enough blocks to reach - // confidence on the message calls - for tid, calls := range newCalls { - for _, data := range calls { - e.queueForConfidence(tid, data, nil, ts) - } - } - - for at := e.lastTs.Height(); at <= ts.Height(); at++ { - // Apply any queued events and timeouts that were targeted at the - // current chain height - e.applyWithConfidence(ts, at) - e.applyTimeouts(ts) - } - - // Update the latest known tipset - e.lastTs = ts } + for at := from.Height() + 1; at <= to.Height(); at++ { + // Apply any queued events and timeouts that were targeted at the + // current chain height + e.applyWithConfidence(ctx, at) + e.applyTimeouts(ctx, to) + } return nil } -func (e *hcEvents) handleReverts(ts *types.TipSet) { - reverts, ok := e.revertQueue[ts.Height()] +func (e *hcEventsObserver) Revert(ctx context.Context, from, to *types.TipSet) error { + e.lk.Lock() + defer e.lk.Unlock() + + defer func() { e.lastTs = to }() + + reverts, ok := e.revertQueue[from.Height()] if !ok { - return // nothing to do + return nil // nothing to do } for _, triggerH := range reverts { - toRevert := e.confQueue[triggerH][ts.Height()] + toRevert := e.confQueue[triggerH][from.Height()] for _, event := range toRevert { if !event.called { continue // event wasn't apply()-ied yet @@ -182,24 +170,21 @@ func (e *hcEvents) handleReverts(ts *types.TipSet) { trigger := e.triggers[event.trigger] - if err := trigger.revert(e.ctx, ts); err != nil { - log.Errorf("reverting chain trigger (@H %d, triggered @ %d) failed: %s", ts.Height(), triggerH, err) + if err := trigger.revert(ctx, from); err != nil { + log.Errorf("reverting chain trigger (@H %d, triggered @ %d) failed: %s", from.Height(), triggerH, err) } } - delete(e.confQueue[triggerH], ts.Height()) + delete(e.confQueue[triggerH], from.Height()) } - delete(e.revertQueue, ts.Height()) + delete(e.revertQueue, from.Height()) + return nil } // Queue up events until the chain has reached a height that reflects the // desired confidence -func (e *hcEvents) queueForConfidence(trigID uint64, data eventData, prevTs, ts *types.TipSet) { +func (e *hcEventsObserver) queueForConfidence(trigID uint64, data eventData, prevTs, ts *types.TipSet) { trigger := e.triggers[trigID] - prevH := NoHeight - if prevTs != nil { - prevH = prevTs.Height() - } appliedH := ts.Height() triggerH := appliedH + abi.ChainEpoch(trigger.confidence) @@ -211,28 +196,23 @@ func (e *hcEvents) queueForConfidence(trigID uint64, data eventData, prevTs, ts } byOrigH[appliedH] = append(byOrigH[appliedH], &queuedEvent{ - trigger: trigID, - prevH: prevH, - h: appliedH, - data: data, + trigger: trigID, + data: data, + tipset: ts, + prevTipset: prevTs, }) e.revertQueue[appliedH] = append(e.revertQueue[appliedH], triggerH) } // Apply any events that were waiting for this chain height for confidence -func (e *hcEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpoch) { +func (e *hcEventsObserver) applyWithConfidence(ctx context.Context, height abi.ChainEpoch) { byOrigH, ok := e.confQueue[height] if !ok { return // no triggers at this height } for origH, events := range byOrigH { - triggerTs, err := e.tsc.get(origH) - if err != nil { - log.Errorf("events: applyWithConfidence didn't find tipset for event; wanted %d; current %d", origH, height) - } - for _, event := range events { if event.called { continue @@ -243,18 +223,7 @@ func (e *hcEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpoch) continue } - // Previous tipset - this is relevant for example in a state change - // from one tipset to another - var prevTs *types.TipSet - if event.prevH != NoHeight { - prevTs, err = e.tsc.get(event.prevH) - if err != nil { - log.Errorf("events: applyWithConfidence didn't find tipset for previous event; wanted %d; current %d", event.prevH, height) - continue - } - } - - more, err := trigger.handle(event.data, prevTs, triggerTs, height) + more, err := trigger.handle(ctx, event.data, event.prevTipset, event.tipset, height) if err != nil { log.Errorf("chain trigger (@H %d, triggered @ %d) failed: %s", origH, height, err) continue // don't revert failed calls @@ -273,7 +242,7 @@ func (e *hcEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpoch) } // Apply any timeouts that expire at this height -func (e *hcEvents) applyTimeouts(ts *types.TipSet) { +func (e *hcEventsObserver) applyTimeouts(ctx context.Context, ts *types.TipSet) { triggers, ok := e.timeouts[ts.Height()] if !ok { return // nothing to do @@ -288,12 +257,13 @@ func (e *hcEvents) applyTimeouts(ts *types.TipSet) { continue } - timeoutTs, err := e.tsc.get(ts.Height() - abi.ChainEpoch(trigger.confidence)) + // This should be cached. + timeoutTs, err := e.cs.ChainGetTipSetAfterHeight(ctx, ts.Height()-abi.ChainEpoch(trigger.confidence), ts.Key()) if err != nil { log.Errorf("events: applyTimeouts didn't find tipset for event; wanted %d; current %d", ts.Height()-abi.ChainEpoch(trigger.confidence), ts.Height()) } - more, err := trigger.handle(nil, nil, timeoutTs, ts.Height()) + more, err := trigger.handle(ctx, nil, nil, timeoutTs, ts.Height()) if err != nil { log.Errorf("chain trigger (call @H %d, called @ %d) failed: %s", timeoutTs.Height(), ts.Height(), err) continue // don't revert failed calls @@ -309,24 +279,24 @@ func (e *hcEvents) applyTimeouts(ts *types.TipSet) { // - RevertHandler: called if the chain head changes causing the event to revert // - confidence: wait this many tipsets before calling EventHandler // - timeout: at this chain height, timeout on waiting for this event -func (e *hcEvents) onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) { +func (e *hcEvents) onHeadChanged(ctx context.Context, check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) { e.lk.Lock() defer e.lk.Unlock() // Check if the event has already occurred - ts, err := e.tsc.best() - if err != nil { - return 0, xerrors.Errorf("error getting best tipset: %w", err) - } - done, more, err := check(ts) - if err != nil { - return 0, xerrors.Errorf("called check error (h: %d): %w", ts.Height(), err) + more := true + done := false + if e.lastTs != nil { + var err error + done, more, err = check(ctx, e.lastTs) + if err != nil { + return 0, xerrors.Errorf("called check error (h: %d): %w", e.lastTs.Height(), err) + } } if done { timeout = NoTimeout } - // Create a trigger for the event id := e.ctr e.ctr++ @@ -354,12 +324,11 @@ func (e *hcEvents) onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHa // headChangeAPI is used to allow the composed event APIs to call back to hcEvents // to listen for changes type headChangeAPI interface { - onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) + onHeadChanged(ctx context.Context, check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) } // watcherEvents watches for a state change type watcherEvents struct { - ctx context.Context cs EventAPI hcAPI headChangeAPI @@ -367,9 +336,8 @@ type watcherEvents struct { matchers map[triggerID]StateMatchFunc } -func newWatcherEvents(ctx context.Context, hcAPI headChangeAPI, cs EventAPI) watcherEvents { +func newWatcherEvents(hcAPI headChangeAPI, cs EventAPI) watcherEvents { return watcherEvents{ - ctx: ctx, cs: cs, hcAPI: hcAPI, matchers: make(map[triggerID]StateMatchFunc), @@ -438,7 +406,7 @@ type StateMatchFunc func(oldTs, newTs *types.TipSet) (bool, StateChange, error) // the state change is queued up until the confidence interval has elapsed (and // `StateChangeHandler` is called) func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf StateMatchFunc) error { - hnd := func(data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) { + hnd := func(ctx context.Context, data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) { states, ok := data.(StateChange) if data != nil && !ok { panic("expected StateChange") @@ -447,7 +415,7 @@ func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler, return scHnd(prevTs, ts, states, height) } - id, err := we.hcAPI.onHeadChanged(check, hnd, rev, confidence, timeout) + id, err := we.hcAPI.onHeadChanged(context.TODO(), check, hnd, rev, confidence, timeout) if err != nil { return err } @@ -461,43 +429,29 @@ func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler, // messageEvents watches for message calls to actors type messageEvents struct { - ctx context.Context cs EventAPI hcAPI headChangeAPI lk sync.RWMutex matchers map[triggerID]MsgMatchFunc - - blockMsgLk sync.Mutex - blockMsgCache *lru.ARCCache } -func newMessageEvents(ctx context.Context, hcAPI headChangeAPI, cs EventAPI) messageEvents { - blsMsgCache, _ := lru.NewARC(500) +func newMessageEvents(hcAPI headChangeAPI, cs EventAPI) messageEvents { return messageEvents{ - ctx: ctx, - cs: cs, - hcAPI: hcAPI, - matchers: make(map[triggerID]MsgMatchFunc), - blockMsgLk: sync.Mutex{}, - blockMsgCache: blsMsgCache, + cs: cs, + hcAPI: hcAPI, + matchers: make(map[triggerID]MsgMatchFunc), } } // Check if there are any new actor calls -func (me *messageEvents) checkNewCalls(ts *types.TipSet) (map[triggerID][]eventData, error) { - pts, err := me.cs.ChainGetTipSet(me.ctx, ts.Parents()) // we actually care about messages in the parent tipset here - if err != nil { - log.Errorf("getting parent tipset in checkNewCalls: %s", err) - return nil, err - } - +func (me *messageEvents) checkNewCalls(ctx context.Context, from, to *types.TipSet) map[triggerID][]eventData { me.lk.RLock() defer me.lk.RUnlock() // For each message in the tipset res := make(map[triggerID][]eventData) - me.messagesForTs(pts, func(msg *types.Message) { + me.messagesForTs(from, func(msg *types.Message) { // TODO: provide receipts // Run each trigger's matcher against the message @@ -516,47 +470,32 @@ func (me *messageEvents) checkNewCalls(ts *types.TipSet) (map[triggerID][]eventD } }) - return res, nil + return res } // Get the messages in a tipset func (me *messageEvents) messagesForTs(ts *types.TipSet, consume func(*types.Message)) { seen := map[cid.Cid]struct{}{} - for _, tsb := range ts.Blocks() { - me.blockMsgLk.Lock() - msgsI, ok := me.blockMsgCache.Get(tsb.Cid()) - var err error - if !ok { - msgsI, err = me.cs.ChainGetBlockMessages(context.TODO(), tsb.Cid()) - if err != nil { - log.Errorf("messagesForTs MessagesForBlock failed (ts.H=%d, Bcid:%s, B.Mcid:%s): %s", ts.Height(), tsb.Cid(), tsb.Messages, err) - // this is quite bad, but probably better than missing all the other updates - me.blockMsgLk.Unlock() - continue - } - me.blockMsgCache.Add(tsb.Cid(), msgsI) + for i, tsb := range ts.Cids() { + msgs, err := me.cs.ChainGetBlockMessages(context.TODO(), tsb) + if err != nil { + log.Errorf("messagesForTs MessagesForBlock failed (ts.H=%d, Bcid:%s, B.Mcid:%s): %s", + ts.Height(), tsb, ts.Blocks()[i].Messages, err) + continue } - me.blockMsgLk.Unlock() - msgs := msgsI.(*api.BlockMessages) - for _, m := range msgs.BlsMessages { - _, ok := seen[m.Cid()] + for i, c := range msgs.Cids { + // We iterate over the CIDs to avoid having to recompute them. + _, ok := seen[c] if ok { continue } - seen[m.Cid()] = struct{}{} - - consume(m) - } - - for _, m := range msgs.SecpkMessages { - _, ok := seen[m.Message.Cid()] - if ok { - continue + seen[c] = struct{}{} + if i < len(msgs.BlsMessages) { + consume(msgs.BlsMessages[i]) + } else { + consume(&msgs.SecpkMessages[i-len(msgs.BlsMessages)].Message) } - seen[m.Message.Cid()] = struct{}{} - - consume(&m.Message) } } } @@ -596,14 +535,14 @@ type MsgMatchFunc func(msg *types.Message) (matched bool, err error) // * `MsgMatchFunc` is called against each message. If there is a match, the // message is queued up until the confidence interval has elapsed (and // `MsgHandler` is called) -func (me *messageEvents) Called(check CheckFunc, msgHnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf MsgMatchFunc) error { - hnd := func(data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) { +func (me *messageEvents) Called(ctx context.Context, check CheckFunc, msgHnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf MsgMatchFunc) error { + hnd := func(ctx context.Context, data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) { msg, ok := data.(*types.Message) if data != nil && !ok { panic("expected msg") } - ml, err := me.cs.StateSearchMsg(me.ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true) + ml, err := me.cs.StateSearchMsg(ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true) if err != nil { return false, err } @@ -615,7 +554,7 @@ func (me *messageEvents) Called(check CheckFunc, msgHnd MsgHandler, rev RevertHa return msgHnd(msg, &ml.Receipt, ts, height) } - id, err := me.hcAPI.onHeadChanged(check, hnd, rev, confidence, timeout) + id, err := me.hcAPI.onHeadChanged(ctx, check, hnd, rev, confidence, timeout) if err != nil { return err } @@ -629,5 +568,5 @@ func (me *messageEvents) Called(check CheckFunc, msgHnd MsgHandler, rev RevertHa // Convenience function for checking and matching messages func (me *messageEvents) CalledMsg(ctx context.Context, hnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, msg types.ChainMsg) error { - return me.Called(me.CheckMsg(ctx, msg, hnd), hnd, rev, confidence, timeout, me.MatchMsg(msg.VMMessage())) + return me.Called(ctx, me.CheckMsg(msg, hnd), hnd, rev, confidence, timeout, me.MatchMsg(msg.VMMessage())) } diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 1fcff9e68..02c252bc9 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -11,136 +11,32 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -type heightEvents struct { - lk sync.Mutex - tsc *tipSetCache - gcConfidence abi.ChainEpoch +type heightHandler struct { + ts *types.TipSet + height abi.ChainEpoch + called bool - ctr triggerID - - heightTriggers map[triggerID]*heightHandler - - htTriggerHeights map[triggerH][]triggerID - htHeights map[msgH][]triggerID - - ctx context.Context + handle HeightHandler + revert RevertHandler } -func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { - ctx, span := trace.StartSpan(e.ctx, "events.HeightHeadChange") - defer span.End() - span.AddAttributes(trace.Int64Attribute("endHeight", int64(app[0].Height()))) - span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev)))) - span.AddAttributes(trace.Int64Attribute("applies", int64(len(app)))) +type heightEvents struct { + api EventAPI + gcConfidence abi.ChainEpoch - e.lk.Lock() - defer e.lk.Unlock() - for _, ts := range rev { - // TODO: log error if h below gcconfidence - // revert height-based triggers + lk sync.Mutex + head *types.TipSet + tsHeights, triggerHeights map[abi.ChainEpoch][]*heightHandler + lastGc abi.ChainEpoch //nolint:structcheck +} - revert := func(h abi.ChainEpoch, ts *types.TipSet) { - for _, tid := range e.htHeights[h] { - ctx, span := trace.StartSpan(ctx, "events.HeightRevert") - - rev := e.heightTriggers[tid].revert - e.lk.Unlock() - err := rev(ctx, ts) - e.lk.Lock() - e.heightTriggers[tid].called = false - - span.End() - - if err != nil { - log.Errorf("reverting chain trigger (@H %d): %s", h, err) - } - } - } - revert(ts.Height(), ts) - - subh := ts.Height() - 1 - for { - cts, err := e.tsc.get(subh) - if err != nil { - return err - } - - if cts != nil { - break - } - - revert(subh, ts) - subh-- - } - - if err := e.tsc.revert(ts); err != nil { - return err - } +func newHeightEvents(api EventAPI, gcConfidence abi.ChainEpoch) *heightEvents { + return &heightEvents{ + api: api, + gcConfidence: gcConfidence, + tsHeights: map[abi.ChainEpoch][]*heightHandler{}, + triggerHeights: map[abi.ChainEpoch][]*heightHandler{}, } - - for i := range app { - ts := app[i] - - if err := e.tsc.add(ts); err != nil { - return err - } - - // height triggers - - apply := func(h abi.ChainEpoch, ts *types.TipSet) error { - for _, tid := range e.htTriggerHeights[h] { - hnd := e.heightTriggers[tid] - if hnd.called { - return nil - } - - triggerH := h - abi.ChainEpoch(hnd.confidence) - - incTs, err := e.tsc.getNonNull(triggerH) - if err != nil { - return err - } - - ctx, span := trace.StartSpan(ctx, "events.HeightApply") - span.AddAttributes(trace.BoolAttribute("immediate", false)) - handle := hnd.handle - e.lk.Unlock() - err = handle(ctx, incTs, h) - e.lk.Lock() - hnd.called = true - span.End() - - if err != nil { - log.Errorf("chain trigger (@H %d, called @ %d) failed: %+v", triggerH, ts.Height(), err) - } - } - return nil - } - - if err := apply(ts.Height(), ts); err != nil { - return err - } - subh := ts.Height() - 1 - for { - cts, err := e.tsc.get(subh) - if err != nil { - return err - } - - if cts != nil { - break - } - - if err := apply(subh, ts); err != nil { - return err - } - - subh-- - } - - } - - return nil } // ChainAt invokes the specified `HeightHandler` when the chain reaches the @@ -148,62 +44,211 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { // specified height, `RevertHandler` will be called. // // ts passed to handlers is the tipset at the specified, or above, if lower tipsets were null -func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence int, h abi.ChainEpoch) error { - e.lk.Lock() // Tricky locking, check your locks if you modify this function! - - best, err := e.tsc.best() - if err != nil { - e.lk.Unlock() - return xerrors.Errorf("error getting best tipset: %w", err) +// +// The context governs cancellations of this call, it won't cancel the event handler. +func (e *heightEvents) ChainAt(ctx context.Context, hnd HeightHandler, rev RevertHandler, confidence int, h abi.ChainEpoch) error { + if abi.ChainEpoch(confidence) > e.gcConfidence { + // Need this to be able to GC effectively. + return xerrors.Errorf("confidence cannot be greater than gcConfidence: %d > %d", confidence, e.gcConfidence) } - - bestH := best.Height() - if bestH >= h+abi.ChainEpoch(confidence) { - ts, err := e.tsc.getNonNull(h) - if err != nil { - log.Warnf("events.ChainAt: calling HandleFunc with nil tipset, not found in cache: %s", err) - } - - e.lk.Unlock() - ctx, span := trace.StartSpan(e.ctx, "events.HeightApply") - span.AddAttributes(trace.BoolAttribute("immediate", true)) - - err = hnd(ctx, ts, bestH) - span.End() - - if err != nil { - return err - } - - e.lk.Lock() - best, err = e.tsc.best() - if err != nil { - e.lk.Unlock() - return xerrors.Errorf("error getting best tipset: %w", err) - } - bestH = best.Height() - } - - defer e.lk.Unlock() - - if bestH >= h+abi.ChainEpoch(confidence)+e.gcConfidence { - return nil - } - - triggerAt := h + abi.ChainEpoch(confidence) - - id := e.ctr - e.ctr++ - - e.heightTriggers[id] = &heightHandler{ - confidence: confidence, - + handler := &heightHandler{ + height: h, handle: hnd, revert: rev, } + triggerAt := h + abi.ChainEpoch(confidence) - e.htHeights[h] = append(e.htHeights[h], id) - e.htTriggerHeights[triggerAt] = append(e.htTriggerHeights[triggerAt], id) + // Here we try to jump onto a moving train. To avoid stopping the train, we release the lock + // while calling the API and/or the trigger functions. Unfortunately, it's entirely possible + // (although unlikely) to go back and forth across the trigger heights, so we need to keep + // going back and forth here till we're synced. + // + // TODO: Consider using a worker goroutine so we can just drop the handler in a channel? The + // downside is that we'd either need a tipset cache, or we'd need to potentially fetch + // tipsets in-line inside the event loop. + e.lk.Lock() + for { + head := e.head + // If we haven't initialized yet, store the trigger and move on. + if head == nil { + e.triggerHeights[triggerAt] = append(e.triggerHeights[triggerAt], handler) + e.tsHeights[h] = append(e.tsHeights[h], handler) + e.lk.Unlock() + return nil + } + + if head.Height() >= h { + // Head is past the handler height. We at least need to stash the tipset to + // avoid doing this from the main event loop. + e.lk.Unlock() + + var ts *types.TipSet + if head.Height() == h { + ts = head + } else { + var err error + ts, err = e.api.ChainGetTipSetAfterHeight(ctx, handler.height, head.Key()) + if err != nil { + return xerrors.Errorf("events.ChainAt: failed to get tipset: %s", err) + } + } + + // If we've applied the handler on the wrong tipset, revert. + if handler.called && !ts.Equals(handler.ts) { + ctx, span := trace.StartSpan(ctx, "events.HeightRevert") + span.AddAttributes(trace.BoolAttribute("immediate", true)) + err := handler.revert(ctx, handler.ts) + span.End() + if err != nil { + return err + } + handler.called = false + } + + // Save the tipset. + handler.ts = ts + + // If we've reached confidence and haven't called, call. + if !handler.called && head.Height() >= triggerAt { + ctx, span := trace.StartSpan(ctx, "events.HeightApply") + span.AddAttributes(trace.BoolAttribute("immediate", true)) + err := handler.handle(ctx, handler.ts, head.Height()) + span.End() + if err != nil { + return err + } + + handler.called = true + + // If we've reached gcConfidence, return without saving anything. + if head.Height() >= h+e.gcConfidence { + return nil + } + } + + e.lk.Lock() + } else if handler.called { + // We're not passed the head (anymore) but have applied the handler. Revert, try again. + e.lk.Unlock() + ctx, span := trace.StartSpan(ctx, "events.HeightRevert") + span.AddAttributes(trace.BoolAttribute("immediate", true)) + err := handler.revert(ctx, handler.ts) + span.End() + if err != nil { + return err + } + handler.called = false + e.lk.Lock() + } // otherwise, we changed heads but the change didn't matter. + + // If we managed to get through this without the head changing, we're finally done. + if head.Equals(e.head) { + e.triggerHeights[triggerAt] = append(e.triggerHeights[triggerAt], handler) + e.tsHeights[h] = append(e.tsHeights[h], handler) + e.lk.Unlock() + return nil + } + } +} + +func (e *heightEvents) observer() TipSetObserver { + return (*heightEventsObserver)(e) +} + +// Updates the head and garbage collects if we're 2x over our garbage collection confidence period. +func (e *heightEventsObserver) updateHead(h *types.TipSet) { + e.lk.Lock() + defer e.lk.Unlock() + e.head = h + + if e.head.Height() < e.lastGc+e.gcConfidence*2 { + return + } + e.lastGc = h.Height() + + targetGcHeight := e.head.Height() - e.gcConfidence + for h := range e.tsHeights { + if h >= targetGcHeight { + continue + } + delete(e.tsHeights, h) + } + for h := range e.triggerHeights { + if h >= targetGcHeight { + continue + } + delete(e.triggerHeights, h) + } +} + +type heightEventsObserver heightEvents + +func (e *heightEventsObserver) Revert(ctx context.Context, from, to *types.TipSet) error { + // Update the head first so we don't accidental skip reverting a concurrent call to ChainAt. + e.updateHead(to) + + // Call revert on all hights between the two tipsets, handling empty tipsets. + for h := from.Height(); h > to.Height(); h-- { + e.lk.Lock() + triggers := e.tsHeights[h] + e.lk.Unlock() + + // 1. Triggers are only invoked from the global event loop, we don't need to hold the lock while calling. + // 2. We only ever append to or replace the trigger slice, so it's safe to iterate over it without the lock. + for _, handler := range triggers { + handler.ts = nil // invalidate + if !handler.called { + // We haven't triggered this yet, or there has been a concurrent call to ChainAt. + continue + } + ctx, span := trace.StartSpan(ctx, "events.HeightRevert") + err := handler.revert(ctx, from) + span.End() + + if err != nil { + log.Errorf("reverting chain trigger (@H %d): %s", h, err) + } + handler.called = false + } + } + return nil +} + +func (e *heightEventsObserver) Apply(ctx context.Context, from, to *types.TipSet) error { + // Update the head first so we don't accidental skip applying a concurrent call to ChainAt. + e.updateHead(to) + + for h := from.Height() + 1; h <= to.Height(); h++ { + e.lk.Lock() + triggers := e.triggerHeights[h] + tipsets := e.tsHeights[h] + e.lk.Unlock() + + // Stash the tipset for future triggers. + for _, handler := range tipsets { + handler.ts = to + } + + // Trigger the ready triggers. + for _, handler := range triggers { + if handler.called { + // We may have reverted past the trigger point, but not past the call point. + // Or there has been a concurrent call to ChainAt. + continue + } + + ctx, span := trace.StartSpan(ctx, "events.HeightApply") + span.AddAttributes(trace.BoolAttribute("immediate", false)) + err := handler.handle(ctx, handler.ts, h) + span.End() + + if err != nil { + log.Errorf("chain trigger (@H %d, called @ %d) failed: %+v", h, to.Height(), err) + } + + handler.called = true + } + } return nil } diff --git a/chain/events/events_test.go b/chain/events/events_test.go index 04f938055..0f4687c8d 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -41,8 +41,6 @@ type fakeCS struct { msgs map[cid.Cid]fakeMsg blkMsgs map[cid.Cid]cid.Cid - sync sync.Mutex - tipsets map[types.TipSetKey]*types.TipSet sub func(rev, app []*types.TipSet) @@ -51,6 +49,20 @@ type fakeCS struct { callNumber map[string]int } +func newFakeCS(t *testing.T) *fakeCS { + fcs := &fakeCS{ + t: t, + h: 1, + msgs: make(map[cid.Cid]fakeMsg), + blkMsgs: make(map[cid.Cid]cid.Cid), + tipsets: make(map[types.TipSetKey]*types.TipSet), + tsc: newTSCache(nil, 2*build.ForkLengthThreshold), + callNumber: map[string]int{}, + } + require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + return fcs +} + func (fcs *fakeCS) ChainHead(ctx context.Context) (*types.TipSet, error) { fcs.callNumberLk.Lock() defer fcs.callNumberLk.Unlock() @@ -58,6 +70,13 @@ func (fcs *fakeCS) ChainHead(ctx context.Context) (*types.TipSet, error) { panic("implement me") } +func (fcs *fakeCS) ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) { + fcs.callNumberLk.Lock() + defer fcs.callNumberLk.Unlock() + fcs.callNumber["ChainGetPath"] = fcs.callNumber["ChainGetPath"] + 1 + panic("Not Implemented") +} + func (fcs *fakeCS) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { fcs.callNumberLk.Lock() defer fcs.callNumberLk.Unlock() @@ -85,6 +104,12 @@ func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types fcs.callNumber["ChainGetTipSetByHeight"] = fcs.callNumber["ChainGetTipSetByHeight"] + 1 panic("Not Implemented") } +func (fcs *fakeCS) ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) { + fcs.callNumberLk.Lock() + defer fcs.callNumberLk.Unlock() + fcs.callNumber["ChainGetTipSetAfterHeight"] = fcs.callNumber["ChainGetTipSetAfterHeight"] + 1 + panic("Not Implemented") +} func (fcs *fakeCS) makeTs(t *testing.T, parents []cid.Cid, h abi.ChainEpoch, msgcid cid.Cid) *types.TipSet { a, _ := address.NewFromString("t00") @@ -132,13 +157,13 @@ func (fcs *fakeCS) makeTs(t *testing.T, parents []cid.Cid, h abi.ChainEpoch, msg return ts } -func (fcs *fakeCS) ChainNotify(context.Context) (<-chan []*api.HeadChange, error) { +func (fcs *fakeCS) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { fcs.callNumberLk.Lock() defer fcs.callNumberLk.Unlock() fcs.callNumber["ChainNotify"] = fcs.callNumber["ChainNotify"] + 1 out := make(chan []*api.HeadChange, 1) - best, err := fcs.tsc.best() + best, err := fcs.tsc.ChainHead(ctx) if err != nil { return nil, err } @@ -160,7 +185,12 @@ func (fcs *fakeCS) ChainNotify(context.Context) (<-chan []*api.HeadChange, error } } - out <- notif + select { + case out <- notif: + case <-ctx.Done(): + // TODO: fail test? + return + } } return out, nil @@ -180,7 +210,15 @@ func (fcs *fakeCS) ChainGetBlockMessages(ctx context.Context, blk cid.Cid) (*api return &api.BlockMessages{}, nil } - return &api.BlockMessages{BlsMessages: ms.bmsgs, SecpkMessages: ms.smsgs}, nil + cids := make([]cid.Cid, len(ms.bmsgs)+len(ms.smsgs)) + for i, m := range ms.bmsgs { + cids[i] = m.Cid() + } + for i, m := range ms.smsgs { + cids[i+len(ms.bmsgs)] = m.Cid() + } + + return &api.BlockMessages{BlsMessages: ms.bmsgs, SecpkMessages: ms.smsgs, Cids: cids}, nil } func (fcs *fakeCS) fakeMsgs(m fakeMsg) cid.Cid { @@ -202,6 +240,9 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / fcs.t.Fatal("sub not be nil") } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nullm := map[int]struct{}{} for _, v := range nulls { nullm[v] = struct{}{} @@ -209,12 +250,14 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / var revs []*types.TipSet for i := 0; i < rev; i++ { - ts, err := fcs.tsc.best() + fcs.t.Log("revert", fcs.h) + from, err := fcs.tsc.ChainHead(ctx) require.NoError(fcs.t, err) - if _, ok := nullm[int(ts.Height())]; !ok { - revs = append(revs, ts) - require.NoError(fcs.t, fcs.tsc.revert(ts)) + if _, ok := nullm[int(from.Height())]; !ok { + revs = append(revs, from) + + require.NoError(fcs.t, fcs.tsc.revert(from)) } fcs.h-- } @@ -222,6 +265,7 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / var apps []*types.TipSet for i := 0; i < app; i++ { fcs.h++ + fcs.t.Log("apply", fcs.h) mc, hasMsgs := msgs[i] if !hasMsgs { @@ -232,7 +276,7 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / continue } - best, err := fcs.tsc.best() + best, err := fcs.tsc.ChainHead(ctx) require.NoError(fcs.t, err) ts := fcs.makeTs(fcs.t, best.Key().Cids(), fcs.h, mc) require.NoError(fcs.t, fcs.tsc.add(ts)) @@ -244,35 +288,24 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / apps = append(apps, ts) } - fcs.sync.Lock() - fcs.sub(revs, apps) - fcs.sync.Lock() - fcs.sync.Unlock() //nolint:staticcheck -} - -func (fcs *fakeCS) notifDone() { - fcs.sync.Unlock() + // Wait for the last round to finish. + fcs.sub(nil, nil) + fcs.sub(nil, nil) } var _ EventAPI = &fakeCS{} func TestAt(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + fcs := newFakeCS(t) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true @@ -325,20 +358,18 @@ func TestAt(t *testing.T) { } func TestAtDoubleTrigger(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - events := NewEvents(context.Background(), fcs) + fcs := newFakeCS(t) + + events, err := NewEvents(ctx, fcs) + require.NoError(t, err) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(ctx, func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true @@ -368,20 +399,14 @@ func TestAtDoubleTrigger(t *testing.T) { } func TestAtNullTrigger(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + fcs := newFakeCS(t) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, abi.ChainEpoch(6), ts.Height()) require.Equal(t, 8, int(curH)) applied = true @@ -403,20 +428,18 @@ func TestAtNullTrigger(t *testing.T) { } func TestAtNullConf(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - events := NewEvents(context.Background(), fcs) + fcs := newFakeCS(t) + + events, err := NewEvents(ctx, fcs) + require.NoError(t, err) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(ctx, func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true @@ -443,22 +466,17 @@ func TestAtNullConf(t *testing.T) { } func TestAtStart(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) fcs.advance(0, 5, nil) // 6 var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true @@ -478,22 +496,17 @@ func TestAtStart(t *testing.T) { } func TestAtStartConfidence(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) fcs.advance(0, 10, nil) // 11 var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 11, int(curH)) applied = true @@ -509,21 +522,16 @@ func TestAtStartConfidence(t *testing.T) { } func TestAtChained(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { - return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + return events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 10, int(ts.Height())) applied = true return nil @@ -544,23 +552,18 @@ func TestAtChained(t *testing.T) { } func TestAtChainedConfidence(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) fcs.advance(0, 15, nil) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { - return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + return events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 10, int(ts.Height())) applied = true return nil @@ -579,22 +582,17 @@ func TestAtChainedConfidence(t *testing.T) { } func TestAtChainedConfidenceNull(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) fcs.advance(0, 15, nil, 5) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { applied = true require.Equal(t, 6, int(ts.Height())) return nil @@ -615,18 +613,10 @@ func matchAddrMethod(to address.Address, m abi.MethodNum) func(msg *types.Messag } func TestCalled(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) @@ -637,7 +627,7 @@ func TestCalled(t *testing.T) { var appliedTs *types.TipSet var appliedH abi.ChainEpoch - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { require.Equal(t, false, applied) @@ -828,25 +818,17 @@ func TestCalled(t *testing.T) { } func TestCalledTimeout(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) called := false - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { called = true @@ -869,20 +851,16 @@ func TestCalledTimeout(t *testing.T) { // with check func reporting done - fcs = &fakeCS{ - t: t, - h: 1, + fcs = newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - callNumber: map[string]int{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + events, err = NewEvents(context.Background(), fcs) + require.NoError(t, err) - events = NewEvents(context.Background(), fcs) + // XXX: Needed to set the latest head so "check" succeeds". Is that OK? Or do we expect + // check to work _before_ we've received any events. + fcs.advance(0, 1, nil) - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return true, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { called = true @@ -904,25 +882,17 @@ func TestCalledTimeout(t *testing.T) { } func TestCalledOrder(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) at := 0 - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { switch at { @@ -968,18 +938,10 @@ func TestCalledOrder(t *testing.T) { } func TestCalledNull(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) @@ -987,7 +949,7 @@ func TestCalledNull(t *testing.T) { more := true var applied, reverted bool - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { require.Equal(t, false, applied) @@ -1034,18 +996,10 @@ func TestCalledNull(t *testing.T) { } func TestRemoveTriggersOnMessage(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) @@ -1053,7 +1007,7 @@ func TestRemoveTriggersOnMessage(t *testing.T) { more := true var applied, reverted bool - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { require.Equal(t, false, applied) @@ -1125,18 +1079,10 @@ type testStateChange struct { } func TestStateChanged(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) more := true var applied, reverted bool @@ -1149,7 +1095,7 @@ func TestStateChanged(t *testing.T) { confidence := 3 timeout := abi.ChainEpoch(20) - err := events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { require.Equal(t, false, applied) @@ -1214,18 +1160,10 @@ func TestStateChanged(t *testing.T) { } func TestStateChangedRevert(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) more := true var applied, reverted bool @@ -1234,7 +1172,7 @@ func TestStateChangedRevert(t *testing.T) { confidence := 1 timeout := abi.ChainEpoch(20) - err := events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { require.Equal(t, false, applied) @@ -1293,22 +1231,14 @@ func TestStateChangedRevert(t *testing.T) { } func TestStateChangedTimeout(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) called := false - err := events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { called = true @@ -1334,20 +1264,15 @@ func TestStateChangedTimeout(t *testing.T) { // with check func reporting done - fcs = &fakeCS{ - t: t, - h: 1, + fcs = newFakeCS(t) + events, err = NewEvents(context.Background(), fcs) + require.NoError(t, err) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - callNumber: map[string]int{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + // XXX: Needed to set the latest head so "check" succeeds". Is that OK? Or do we expect + // check to work _before_ we've received any events. + fcs.advance(0, 1, nil) - events = NewEvents(context.Background(), fcs) - - err = events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return true, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { called = true @@ -1371,25 +1296,17 @@ func TestStateChangedTimeout(t *testing.T) { } func TestCalledMultiplePerEpoch(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - callNumber: map[string]int{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) at := 0 - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { switch at { @@ -1431,18 +1348,10 @@ func TestCalledMultiplePerEpoch(t *testing.T) { } func TestCachedSameBlock(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - callNumber: map[string]int{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - _ = NewEvents(context.Background(), fcs) + _, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) fcs.advance(0, 10, map[int]cid.Cid{}) assert.Assert(t, fcs.callNumber["ChainGetBlockMessages"] == 20, "expect call ChainGetBlockMessages %d but got ", 20, fcs.callNumber["ChainGetBlockMessages"]) diff --git a/chain/events/message_cache.go b/chain/events/message_cache.go new file mode 100644 index 000000000..75e179ad9 --- /dev/null +++ b/chain/events/message_cache.go @@ -0,0 +1,42 @@ +package events + +import ( + "context" + "sync" + + "github.com/filecoin-project/lotus/api" + lru "github.com/hashicorp/golang-lru" + "github.com/ipfs/go-cid" +) + +type messageCache struct { + api EventAPI + + blockMsgLk sync.Mutex + blockMsgCache *lru.ARCCache +} + +func newMessageCache(api EventAPI) *messageCache { + blsMsgCache, _ := lru.NewARC(500) + + return &messageCache{ + api: api, + blockMsgCache: blsMsgCache, + } +} + +func (c *messageCache) ChainGetBlockMessages(ctx context.Context, blkCid cid.Cid) (*api.BlockMessages, error) { + c.blockMsgLk.Lock() + defer c.blockMsgLk.Unlock() + + msgsI, ok := c.blockMsgCache.Get(blkCid) + var err error + if !ok { + msgsI, err = c.api.ChainGetBlockMessages(ctx, blkCid) + if err != nil { + return nil, err + } + c.blockMsgCache.Add(blkCid, msgsI) + } + return msgsI.(*api.BlockMessages), nil +} diff --git a/chain/events/observer.go b/chain/events/observer.go new file mode 100644 index 000000000..cd25b4874 --- /dev/null +++ b/chain/events/observer.go @@ -0,0 +1,234 @@ +package events + +import ( + "context" + "sync" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "go.opencensus.io/trace" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" +) + +type observer struct { + api EventAPI + + lk sync.Mutex + gcConfidence abi.ChainEpoch + + ready chan struct{} + + head *types.TipSet + maxHeight abi.ChainEpoch + + observers []TipSetObserver +} + +func newObserver(api EventAPI, gcConfidence abi.ChainEpoch) *observer { + return &observer{ + api: api, + gcConfidence: gcConfidence, + + ready: make(chan struct{}), + observers: []TipSetObserver{}, + } +} + +func (o *observer) start(ctx context.Context) error { + go o.listenHeadChanges(ctx) + + // Wait for the first tipset to be seen or bail if shutting down + select { + case <-o.ready: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (o *observer) listenHeadChanges(ctx context.Context) { + for { + if err := o.listenHeadChangesOnce(ctx); err != nil { + log.Errorf("listen head changes errored: %s", err) + } else { + log.Warn("listenHeadChanges quit") + } + select { + case <-build.Clock.After(time.Second): + case <-ctx.Done(): + log.Warnf("not restarting listenHeadChanges: context error: %s", ctx.Err()) + return + } + + log.Info("restarting listenHeadChanges") + } +} + +func (o *observer) listenHeadChangesOnce(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + notifs, err := o.api.ChainNotify(ctx) + if err != nil { + // Retry is handled by caller + return xerrors.Errorf("listenHeadChanges ChainNotify call failed: %w", err) + } + + var cur []*api.HeadChange + var ok bool + + // Wait for first tipset or bail + select { + case cur, ok = <-notifs: + if !ok { + return xerrors.Errorf("notification channel closed") + } + case <-ctx.Done(): + return ctx.Err() + } + + if len(cur) != 1 { + return xerrors.Errorf("unexpected initial head notification length: %d", len(cur)) + } + + if cur[0].Type != store.HCCurrent { + return xerrors.Errorf("expected first head notification type to be 'current', was '%s'", cur[0].Type) + } + + head := cur[0].Val + if o.head == nil { + o.head = head + close(o.ready) + } else if !o.head.Equals(head) { + changes, err := o.api.ChainGetPath(ctx, o.head.Key(), head.Key()) + if err != nil { + return xerrors.Errorf("failed to get path from last applied tipset to head: %w", err) + } + + if err := o.applyChanges(ctx, changes); err != nil { + return xerrors.Errorf("failed to apply head changes: %w", err) + } + } + + for changes := range notifs { + if err := o.applyChanges(ctx, changes); err != nil { + return err + } + } + + return nil +} + +func (o *observer) applyChanges(ctx context.Context, changes []*api.HeadChange) error { + // Used to wait for a prior notification round to finish (by tests) + if len(changes) == 0 { + return nil + } + + var rev, app []*types.TipSet + for _, changes := range changes { + switch changes.Type { + case store.HCRevert: + rev = append(rev, changes.Val) + case store.HCApply: + app = append(app, changes.Val) + default: + log.Errorf("unexpected head change notification type: '%s'", changes.Type) + } + } + + if err := o.headChange(ctx, rev, app); err != nil { + return xerrors.Errorf("failed to apply head changes: %w", err) + } + return nil +} + +func (o *observer) headChange(ctx context.Context, rev, app []*types.TipSet) error { + ctx, span := trace.StartSpan(ctx, "events.HeadChange") + span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev)))) + span.AddAttributes(trace.Int64Attribute("applies", int64(len(app)))) + defer func() { + span.AddAttributes(trace.Int64Attribute("endHeight", int64(o.head.Height()))) + span.End() + }() + + // NOTE: bailing out here if the head isn't what we expected is fine. We'll re-start the + // entire process and handle any strange reorgs. + for i, from := range rev { + if !from.Equals(o.head) { + return xerrors.Errorf( + "expected to revert %s (%d), reverting %s (%d)", + o.head.Key(), o.head.Height(), from.Key(), from.Height(), + ) + } + var to *types.TipSet + if i+1 < len(rev) { + // If we have more reverts, the next revert is the next head. + to = rev[i+1] + } else { + // At the end of the revert sequenece, we need to looup the joint tipset + // between the revert sequence and the apply sequence. + var err error + to, err = o.api.ChainGetTipSet(ctx, from.Parents()) + if err != nil { + // Well, this sucks. We'll bail and restart. + return xerrors.Errorf("failed to get tipset when reverting due to a SetHeead: %w", err) + } + } + + // Get the observers late in case an observer registers/unregisters itself. + o.lk.Lock() + observers := o.observers + o.lk.Unlock() + + for _, obs := range observers { + if err := obs.Revert(ctx, from, to); err != nil { + log.Errorf("observer %T failed to apply tipset %s (%d) with: %s", obs, from.Key(), from.Height(), err) + } + } + + if to.Height() < o.maxHeight-o.gcConfidence { + log.Errorf("reverted past finality, from %d to %d", o.maxHeight, to.Height()) + } + + o.head = to + } + + for _, to := range app { + if to.Parents() != o.head.Key() { + return xerrors.Errorf( + "cannot apply %s (%d) with parents %s on top of %s (%d)", + to.Key(), to.Height(), to.Parents(), o.head.Key(), o.head.Height(), + ) + } + + // Get the observers late in case an observer registers/unregisters itself. + o.lk.Lock() + observers := o.observers + o.lk.Unlock() + + for _, obs := range observers { + if err := obs.Apply(ctx, o.head, to); err != nil { + log.Errorf("observer %T failed to revert tipset %s (%d) with: %s", obs, to.Key(), to.Height(), err) + } + } + o.head = to + if to.Height() > o.maxHeight { + o.maxHeight = to.Height() + } + + } + return nil +} + +// TODO: add a confidence level so we can have observers with difference levels of confidence +func (o *observer) Observe(obs TipSetObserver) { + o.lk.Lock() + defer o.lk.Unlock() + o.observers = append(o.observers, obs) +} diff --git a/chain/events/tscache.go b/chain/events/tscache.go index 7beb80364..033c72a22 100644 --- a/chain/events/tscache.go +++ b/chain/events/tscache.go @@ -11,7 +11,9 @@ import ( ) type tsCacheAPI interface { + ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) + ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error) ChainHead(context.Context) (*types.TipSet, error) } @@ -20,68 +22,157 @@ type tsCacheAPI interface { type tipSetCache struct { mu sync.RWMutex - cache []*types.TipSet - start int // chain head (end) - len int + byKey map[types.TipSetKey]*types.TipSet + byHeight []*types.TipSet + start int // chain head (end) + len int storage tsCacheAPI } -func newTSCache(cap abi.ChainEpoch, storage tsCacheAPI) *tipSetCache { +func newTSCache(storage tsCacheAPI, cap abi.ChainEpoch) *tipSetCache { return &tipSetCache{ - cache: make([]*types.TipSet, cap), - start: 0, - len: 0, + byKey: make(map[types.TipSetKey]*types.TipSet, cap), + byHeight: make([]*types.TipSet, cap), + start: 0, + len: 0, storage: storage, } } +func (tsc *tipSetCache) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + if ts, ok := tsc.byKey[tsk]; ok { + return ts, nil + } + return tsc.storage.ChainGetTipSet(ctx, tsk) +} -func (tsc *tipSetCache) add(ts *types.TipSet) error { +func (tsc *tipSetCache) ChainGetTipSetByHeight(ctx context.Context, height abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + return tsc.get(ctx, height, tsk, true) +} + +func (tsc *tipSetCache) ChainGetTipSetAfterHeight(ctx context.Context, height abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + return tsc.get(ctx, height, tsk, false) +} + +func (tsc *tipSetCache) get(ctx context.Context, height abi.ChainEpoch, tsk types.TipSetKey, prev bool) (*types.TipSet, error) { + fallback := tsc.storage.ChainGetTipSetAfterHeight + if prev { + fallback = tsc.storage.ChainGetTipSetByHeight + } + tsc.mu.RLock() + + // Nothing in the cache? + if tsc.len == 0 { + tsc.mu.RUnlock() + log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height) + return fallback(ctx, height, tsk) + } + + // Resolve the head. + head := tsc.byHeight[tsc.start] + if !tsk.IsEmpty() { + // Not on this chain? + var ok bool + head, ok = tsc.byKey[tsk] + if !ok { + tsc.mu.RUnlock() + return fallback(ctx, height, tsk) + } + } + + headH := head.Height() + tailH := headH - abi.ChainEpoch(tsc.len) + + if headH == height { + tsc.mu.RUnlock() + return head, nil + } else if headH < height { + tsc.mu.RUnlock() + // If the user doesn't pass a tsk, we assume "head" is the last tipset we processed. + return nil, xerrors.Errorf("requested epoch is in the future") + } else if height < tailH { + log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tailH) + tsc.mu.RUnlock() + return fallback(ctx, height, head.Key()) + } + + direction := 1 + if prev { + direction = -1 + } + var ts *types.TipSet + for i := 0; i < tsc.len && ts == nil; i += direction { + ts = tsc.byHeight[normalModulo(tsc.start-int(headH-height)+i, len(tsc.byHeight))] + } + tsc.mu.RUnlock() + return ts, nil +} + +func (tsc *tipSetCache) ChainHead(ctx context.Context) (*types.TipSet, error) { + tsc.mu.RLock() + best := tsc.byHeight[tsc.start] + tsc.mu.RUnlock() + if best == nil { + return tsc.storage.ChainHead(ctx) + } + return best, nil +} + +func (tsc *tipSetCache) add(to *types.TipSet) error { tsc.mu.Lock() defer tsc.mu.Unlock() if tsc.len > 0 { - best := tsc.cache[tsc.start] - if best.Height() >= ts.Height() { - return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.cache[tsc.start].Height()+1, ts.Height()) + best := tsc.byHeight[tsc.start] + if best.Height() >= to.Height() { + return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.byHeight[tsc.start].Height()+1, to.Height()) } - if best.Key() != ts.Parents() { + if best.Key() != to.Parents() { return xerrors.Errorf( "tipSetCache.add: expected new tipset %s (%d) to follow %s (%d), its parents are %s", - ts.Key(), ts.Height(), best.Key(), best.Height(), best.Parents(), + to.Key(), to.Height(), best.Key(), best.Height(), best.Parents(), ) } } - nextH := ts.Height() + nextH := to.Height() if tsc.len > 0 { - nextH = tsc.cache[tsc.start].Height() + 1 + nextH = tsc.byHeight[tsc.start].Height() + 1 } // fill null blocks - for nextH != ts.Height() { - tsc.start = normalModulo(tsc.start+1, len(tsc.cache)) - tsc.cache[tsc.start] = nil - if tsc.len < len(tsc.cache) { + for nextH != to.Height() { + tsc.start = normalModulo(tsc.start+1, len(tsc.byHeight)) + was := tsc.byHeight[tsc.start] + if was != nil { + tsc.byHeight[tsc.start] = nil + delete(tsc.byKey, was.Key()) + } + if tsc.len < len(tsc.byHeight) { tsc.len++ } nextH++ } - tsc.start = normalModulo(tsc.start+1, len(tsc.cache)) - tsc.cache[tsc.start] = ts - if tsc.len < len(tsc.cache) { + tsc.start = normalModulo(tsc.start+1, len(tsc.byHeight)) + was := tsc.byHeight[tsc.start] + if was != nil { + delete(tsc.byKey, was.Key()) + } + tsc.byHeight[tsc.start] = to + if tsc.len < len(tsc.byHeight) { tsc.len++ } + tsc.byKey[to.Key()] = to return nil } -func (tsc *tipSetCache) revert(ts *types.TipSet) error { +func (tsc *tipSetCache) revert(from *types.TipSet) error { tsc.mu.Lock() defer tsc.mu.Unlock() - return tsc.revertUnlocked(ts) + return tsc.revertUnlocked(from) } func (tsc *tipSetCache) revertUnlocked(ts *types.TipSet) error { @@ -89,75 +180,35 @@ func (tsc *tipSetCache) revertUnlocked(ts *types.TipSet) error { return nil // this can happen, and it's fine } - if !tsc.cache[tsc.start].Equals(ts) { + was := tsc.byHeight[tsc.start] + + if !was.Equals(ts) { return xerrors.New("tipSetCache.revert: revert tipset didn't match cache head") } + delete(tsc.byKey, was.Key()) - tsc.cache[tsc.start] = nil - tsc.start = normalModulo(tsc.start-1, len(tsc.cache)) + tsc.byHeight[tsc.start] = nil + tsc.start = normalModulo(tsc.start-1, len(tsc.byHeight)) tsc.len-- _ = tsc.revertUnlocked(nil) // revert null block gap return nil } -func (tsc *tipSetCache) getNonNull(height abi.ChainEpoch) (*types.TipSet, error) { - for { - ts, err := tsc.get(height) - if err != nil { - return nil, err - } - if ts != nil { - return ts, nil - } - height++ - } +func (tsc *tipSetCache) observer() TipSetObserver { + return (*tipSetCacheObserver)(tsc) } -func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { - tsc.mu.RLock() +type tipSetCacheObserver tipSetCache - if tsc.len == 0 { - tsc.mu.RUnlock() - log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height) - return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, types.EmptyTSK) - } +var _ TipSetObserver = new(tipSetCacheObserver) - headH := tsc.cache[tsc.start].Height() - - if height > headH { - tsc.mu.RUnlock() - return nil, xerrors.Errorf("tipSetCache.get: requested tipset not in cache (req: %d, cache head: %d)", height, headH) - } - - clen := len(tsc.cache) - var tail *types.TipSet - for i := 1; i <= tsc.len; i++ { - tail = tsc.cache[normalModulo(tsc.start-tsc.len+i, clen)] - if tail != nil { - break - } - } - - if height < tail.Height() { - tsc.mu.RUnlock() - log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tail.Height()) - return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, tail.Key()) - } - - ts := tsc.cache[normalModulo(tsc.start-int(headH-height), clen)] - tsc.mu.RUnlock() - return ts, nil +func (tsc *tipSetCacheObserver) Apply(_ context.Context, _, to *types.TipSet) error { + return (*tipSetCache)(tsc).add(to) } -func (tsc *tipSetCache) best() (*types.TipSet, error) { - tsc.mu.RLock() - best := tsc.cache[tsc.start] - tsc.mu.RUnlock() - if best == nil { - return tsc.storage.ChainHead(context.TODO()) - } - return best, nil +func (tsc *tipSetCacheObserver) Revert(ctx context.Context, from, _ *types.TipSet) error { + return (*tipSetCache)(tsc).revert(from) } func normalModulo(n, m int) int { diff --git a/chain/events/tscache_test.go b/chain/events/tscache_test.go index 9ba9a556c..c3779eb9e 100644 --- a/chain/events/tscache_test.go +++ b/chain/events/tscache_test.go @@ -17,6 +17,11 @@ type tsCacheAPIFailOnStorageCall struct { t *testing.T } +func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSetAfterHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { + tc.t.Fatal("storage call") + return &types.TipSet{}, nil +} + func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { tc.t.Fatal("storage call") return &types.TipSet{}, nil @@ -25,6 +30,10 @@ func (tc *tsCacheAPIFailOnStorageCall) ChainHead(ctx context.Context) (*types.Ti tc.t.Fatal("storage call") return &types.TipSet{}, nil } +func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + tc.t.Fatal("storage call") + return &types.TipSet{}, nil +} type cacheHarness struct { t *testing.T @@ -40,7 +49,7 @@ func newCacheharness(t *testing.T) *cacheHarness { h := &cacheHarness{ t: t, - tsc: newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t}), + tsc: newTSCache(&tsCacheAPIFailOnStorageCall{t: t}, 50), height: 75, miner: a, } @@ -65,13 +74,13 @@ func (h *cacheHarness) addWithParents(parents []cid.Cid) { } func (h *cacheHarness) add() { - last, err := h.tsc.best() + last, err := h.tsc.ChainHead(context.Background()) require.NoError(h.t, err) h.addWithParents(last.Cids()) } func (h *cacheHarness) revert() { - best, err := h.tsc.best() + best, err := h.tsc.ChainHead(context.Background()) require.NoError(h.t, err) err = h.tsc.revert(best) require.NoError(h.t, err) @@ -95,6 +104,7 @@ func TestTsCache(t *testing.T) { } func TestTsCacheNulls(t *testing.T) { + ctx := context.Background() h := newCacheharness(t) h.add() @@ -105,66 +115,77 @@ func TestTsCacheNulls(t *testing.T) { h.add() h.add() - best, err := h.tsc.best() + best, err := h.tsc.ChainHead(ctx) require.NoError(t, err) require.Equal(t, h.height-1, best.Height()) - ts, err := h.tsc.get(h.height - 1) + ts, err := h.tsc.ChainGetTipSetByHeight(ctx, h.height-1, types.EmptyTSK) require.NoError(t, err) require.Equal(t, h.height-1, ts.Height()) - ts, err = h.tsc.get(h.height - 2) + ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-2, types.EmptyTSK) require.NoError(t, err) require.Equal(t, h.height-2, ts.Height()) - ts, err = h.tsc.get(h.height - 3) - require.NoError(t, err) - require.Nil(t, ts) - - ts, err = h.tsc.get(h.height - 8) + // Should skip the nulls and walk back to the last tipset. + ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-3, types.EmptyTSK) require.NoError(t, err) require.Equal(t, h.height-8, ts.Height()) - best, err = h.tsc.best() + ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-8, types.EmptyTSK) + require.NoError(t, err) + require.Equal(t, h.height-8, ts.Height()) + + best, err = h.tsc.ChainHead(ctx) require.NoError(t, err) require.NoError(t, h.tsc.revert(best)) - best, err = h.tsc.best() + best, err = h.tsc.ChainHead(ctx) require.NoError(t, err) require.NoError(t, h.tsc.revert(best)) - best, err = h.tsc.best() + best, err = h.tsc.ChainHead(ctx) require.NoError(t, err) require.Equal(t, h.height-8, best.Height()) h.skip(50) h.add() - ts, err = h.tsc.get(h.height - 1) + ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-1, types.EmptyTSK) require.NoError(t, err) require.Equal(t, h.height-1, ts.Height()) } type tsCacheAPIStorageCallCounter struct { - t *testing.T - chainGetTipSetByHeight int - chainHead int + t *testing.T + chainGetTipSetByHeight int + chainGetTipSetAfterHeight int + chainGetTipSet int + chainHead int } func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { tc.chainGetTipSetByHeight++ return &types.TipSet{}, nil } +func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSetAfterHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { + tc.chainGetTipSetAfterHeight++ + return &types.TipSet{}, nil +} func (tc *tsCacheAPIStorageCallCounter) ChainHead(ctx context.Context) (*types.TipSet, error) { tc.chainHead++ return &types.TipSet{}, nil } +func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + tc.chainGetTipSet++ + return &types.TipSet{}, nil +} func TestTsCacheEmpty(t *testing.T) { // Calling best on an empty cache should just call out to the chain API callCounter := &tsCacheAPIStorageCallCounter{t: t} - tsc := newTSCache(50, callCounter) - _, err := tsc.best() + tsc := newTSCache(callCounter, 50) + _, err := tsc.ChainHead(context.Background()) require.NoError(t, err) require.Equal(t, 1, callCounter.chainHead) } diff --git a/chain/events/utils.go b/chain/events/utils.go index 91ea0cd7a..0bfb58e0a 100644 --- a/chain/events/utils.go +++ b/chain/events/utils.go @@ -10,10 +10,10 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -func (me *messageEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd MsgHandler) CheckFunc { +func (me *messageEvents) CheckMsg(smsg types.ChainMsg, hnd MsgHandler) CheckFunc { msg := smsg.VMMessage() - return func(ts *types.TipSet) (done bool, more bool, err error) { + return func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { fa, err := me.cs.StateGetActor(ctx, msg.From, ts.Key()) if err != nil { return false, true, err @@ -24,7 +24,7 @@ func (me *messageEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd return false, true, nil } - ml, err := me.cs.StateSearchMsg(me.ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true) + ml, err := me.cs.StateSearchMsg(ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true) if err != nil { return false, true, xerrors.Errorf("getting receipt in CheckMsg: %w", err) } diff --git a/itests/paych_api_test.go b/itests/paych_api_test.go index 647db21e0..49c23545b 100644 --- a/itests/paych_api_test.go +++ b/itests/paych_api_test.go @@ -103,10 +103,11 @@ func TestPaymentChannelsAPI(t *testing.T) { creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(paymentCreator))) // wait for the receiver to submit their vouchers - ev := events.NewEvents(ctx, paymentCreator) + ev, err := events.NewEvents(ctx, paymentCreator) + require.NoError(t, err) preds := state.NewStatePredicates(paymentCreator) finished := make(chan struct{}) - err = ev.StateChanged(func(ts *types.TipSet) (done bool, more bool, err error) { + err = ev.StateChanged(func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { act, err := paymentCreator.StateGetActor(ctx, channel, ts.Key()) if err != nil { return false, false, err diff --git a/itests/paych_cli_test.go b/itests/paych_cli_test.go index 17e7bcbf6..a4ad1920b 100644 --- a/itests/paych_cli_test.go +++ b/itests/paych_cli_test.go @@ -381,8 +381,9 @@ func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) { // waitForHeight waits for the node to reach the given chain epoch func waitForHeight(ctx context.Context, t *testing.T, node kit.TestFullNode, height abi.ChainEpoch) { atHeight := make(chan struct{}) - chainEvents := events.NewEvents(ctx, node) - err := chainEvents.ChainAt(func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + chainEvents, err := events.NewEvents(ctx, node) + require.NoError(t, err) + err = chainEvents.ChainAt(ctx, func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { close(atHeight) return nil }, nil, 1, height) diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 80ead2be3..2ffa56f5f 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -50,11 +50,14 @@ type clientApi struct { full.MpoolAPI } -func NewClientNodeAdapter(mctx helpers.MetricsCtx, lc fx.Lifecycle, stateapi full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, fundmgr *market.FundManager) storagemarket.StorageClientNode { +func NewClientNodeAdapter(mctx helpers.MetricsCtx, lc fx.Lifecycle, stateapi full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, fundmgr *market.FundManager) (storagemarket.StorageClientNode, error) { capi := &clientApi{chain, stateapi, mpool} ctx := helpers.LifecycleCtx(mctx, lc) - ev := events.NewEvents(ctx, capi) + ev, err := events.NewEvents(ctx, capi) + if err != nil { + return nil, err + } a := &ClientNodeAdapter{ clientApi: capi, @@ -63,7 +66,7 @@ func NewClientNodeAdapter(mctx helpers.MetricsCtx, lc fx.Lifecycle, stateapi ful dsMatcher: newDealStateMatcher(state.NewStatePredicates(state.WrapFastAPI(capi))), } a.scMgr = NewSectorCommittedManager(ev, a, &apiWrapper{api: capi}) - return a + return a, nil } func (c *ClientNodeAdapter) ListStorageProviders(ctx context.Context, encodedTs shared.TipSetToken) ([]*storagemarket.StorageProviderInfo, error) { @@ -262,7 +265,7 @@ func (c *ClientNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID a } // Called immediately to check if the deal has already expired or been slashed - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + checkFunc := func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { if ts == nil { // keep listening for events return false, true, nil diff --git a/markets/storageadapter/ondealsectorcommitted.go b/markets/storageadapter/ondealsectorcommitted.go index 31bc0b8bf..4cd0a2d68 100644 --- a/markets/storageadapter/ondealsectorcommitted.go +++ b/markets/storageadapter/ondealsectorcommitted.go @@ -22,7 +22,7 @@ import ( ) type eventsCalledAPI interface { - Called(check events.CheckFunc, msgHnd events.MsgHandler, rev events.RevertHandler, confidence int, timeout abi.ChainEpoch, mf events.MsgMatchFunc) error + Called(ctx context.Context, check events.CheckFunc, msgHnd events.MsgHandler, rev events.RevertHandler, confidence int, timeout abi.ChainEpoch, mf events.MsgMatchFunc) error } type dealInfoAPI interface { @@ -64,7 +64,7 @@ func (mgr *SectorCommittedManager) OnDealSectorPreCommitted(ctx context.Context, } // First check if the deal is already active, and if so, bail out - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + checkFunc := func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { dealInfo, isActive, err := mgr.checkIfDealAlreadyActive(ctx, ts, &proposal, publishCid) if err != nil { // Note: the error returned from here will end up being returned @@ -165,7 +165,7 @@ func (mgr *SectorCommittedManager) OnDealSectorPreCommitted(ctx context.Context, return nil } - if err := mgr.ev.Called(checkFunc, called, revert, int(build.MessageConfidence+1), timeoutEpoch, matchEvent); err != nil { + if err := mgr.ev.Called(ctx, checkFunc, called, revert, int(build.MessageConfidence+1), timeoutEpoch, matchEvent); err != nil { return xerrors.Errorf("failed to set up called handler: %w", err) } @@ -182,7 +182,7 @@ func (mgr *SectorCommittedManager) OnDealSectorCommitted(ctx context.Context, pr } // First check if the deal is already active, and if so, bail out - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + checkFunc := func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { _, isActive, err := mgr.checkIfDealAlreadyActive(ctx, ts, &proposal, publishCid) if err != nil { // Note: the error returned from here will end up being returned @@ -257,7 +257,7 @@ func (mgr *SectorCommittedManager) OnDealSectorCommitted(ctx context.Context, pr return nil } - if err := mgr.ev.Called(checkFunc, called, revert, int(build.MessageConfidence+1), timeoutEpoch, matchEvent); err != nil { + if err := mgr.ev.Called(ctx, checkFunc, called, revert, int(build.MessageConfidence+1), timeoutEpoch, matchEvent); err != nil { return xerrors.Errorf("failed to set up called handler: %w", err) } diff --git a/markets/storageadapter/ondealsectorcommitted_test.go b/markets/storageadapter/ondealsectorcommitted_test.go index db56ee651..86c01799a 100644 --- a/markets/storageadapter/ondealsectorcommitted_test.go +++ b/markets/storageadapter/ondealsectorcommitted_test.go @@ -477,13 +477,13 @@ type fakeEvents struct { DealStartEpochTimeout bool } -func (fe *fakeEvents) Called(check events.CheckFunc, msgHnd events.MsgHandler, rev events.RevertHandler, confidence int, timeout abi.ChainEpoch, mf events.MsgMatchFunc) error { +func (fe *fakeEvents) Called(ctx context.Context, check events.CheckFunc, msgHnd events.MsgHandler, rev events.RevertHandler, confidence int, timeout abi.ChainEpoch, mf events.MsgMatchFunc) error { if fe.DealStartEpochTimeout { msgHnd(nil, nil, nil, 100) // nolint:errcheck return nil } - _, more, err := check(fe.CheckTs) + _, more, err := check(ctx, fe.CheckTs) if err != nil { return err } @@ -506,7 +506,7 @@ func (fe *fakeEvents) Called(check events.CheckFunc, msgHnd events.MsgHandler, r return nil } if matchMessage.doesRevert { - err := rev(fe.Ctx, matchMessage.ts) + err := rev(ctx, matchMessage.ts) if err != nil { return err } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 23a3c32a8..5c82d0dc8 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -55,11 +55,14 @@ type ProviderNodeAdapter struct { scMgr *SectorCommittedManager } -func NewProviderNodeAdapter(fc *config.MinerFeeConfig, dc *config.DealmakingConfig) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, secb *sectorblocks.SectorBlocks, full v1api.FullNode, dealPublisher *DealPublisher) storagemarket.StorageProviderNode { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, secb *sectorblocks.SectorBlocks, full v1api.FullNode, dealPublisher *DealPublisher) storagemarket.StorageProviderNode { +func NewProviderNodeAdapter(fc *config.MinerFeeConfig, dc *config.DealmakingConfig) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, secb *sectorblocks.SectorBlocks, full v1api.FullNode, dealPublisher *DealPublisher) (storagemarket.StorageProviderNode, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, secb *sectorblocks.SectorBlocks, full v1api.FullNode, dealPublisher *DealPublisher) (storagemarket.StorageProviderNode, error) { ctx := helpers.LifecycleCtx(mctx, lc) - ev := events.NewEvents(ctx, full) + ev, err := events.NewEvents(ctx, full) + if err != nil { + return nil, err + } na := &ProviderNodeAdapter{ FullNode: full, @@ -77,7 +80,7 @@ func NewProviderNodeAdapter(fc *config.MinerFeeConfig, dc *config.DealmakingConf } na.scMgr = NewSectorCommittedManager(ev, na, &apiWrapper{api: full}) - return na + return na, nil } } @@ -340,7 +343,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID } // Called immediately to check if the deal has already expired or been slashed - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + checkFunc := func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { if ts == nil { // keep listening for events return false, true, nil diff --git a/paychmgr/settler/settler.go b/paychmgr/settler/settler.go index ce31ab223..38fcc3b92 100644 --- a/paychmgr/settler/settler.go +++ b/paychmgr/settler/settler.go @@ -56,8 +56,11 @@ func SettlePaymentChannels(mctx helpers.MetricsCtx, lc fx.Lifecycle, papi API) e lc.Append(fx.Hook{ OnStart: func(context.Context) error { pcs := newPaymentChannelSettler(ctx, &papi) - ev := events.NewEvents(ctx, papi) - return ev.Called(pcs.check, pcs.messageHandler, pcs.revertHandler, int(build.MessageConfidence+1), events.NoTimeout, pcs.matcher) + ev, err := events.NewEvents(ctx, &papi) + if err != nil { + return err + } + return ev.Called(ctx, pcs.check, pcs.messageHandler, pcs.revertHandler, int(build.MessageConfidence+1), events.NoTimeout, pcs.matcher) }, }) return nil @@ -70,7 +73,7 @@ func newPaymentChannelSettler(ctx context.Context, api settlerAPI) *paymentChann } } -func (pcs *paymentChannelSettler) check(ts *types.TipSet) (done bool, more bool, err error) { +func (pcs *paymentChannelSettler) check(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { return false, true, nil } diff --git a/storage/adapter_events.go b/storage/adapter_events.go index ff69c1e51..0f9c62039 100644 --- a/storage/adapter_events.go +++ b/storage/adapter_events.go @@ -21,7 +21,7 @@ func NewEventsAdapter(api *events.Events) EventsAdapter { } func (e EventsAdapter) ChainAt(hnd sealing.HeightHandler, rev sealing.RevertHandler, confidence int, h abi.ChainEpoch) error { - return e.delegate.ChainAt(func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + return e.delegate.ChainAt(context.TODO(), func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { return hnd(ctx, ts.Key().Bytes(), curH) }, func(ctx context.Context, ts *types.TipSet) error { return rev(ctx, ts.Key().Bytes()) diff --git a/storage/miner.go b/storage/miner.go index c4bf41c12..155f5f30d 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -116,6 +116,7 @@ type fullNodeFilteredAPI interface { ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) + ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) ChainHasObj(context.Context, cid.Cid) (bool, error) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) @@ -167,28 +168,29 @@ func (m *Miner) Run(ctx context.Context) error { return xerrors.Errorf("getting miner info: %w", err) } - var ( - // consumer of chain head changes. - evts = events.NewEvents(ctx, m.api) - evtsAdapter = NewEventsAdapter(evts) + // consumer of chain head changes. + evts, err := events.NewEvents(ctx, m.api) + if err != nil { + return xerrors.Errorf("failed to subscribe to events: %w", err) + } + evtsAdapter := NewEventsAdapter(evts) - // Create a shim to glue the API required by the sealing component - // with the API that Lotus is capable of providing. - // The shim translates between "tipset tokens" and tipset keys, and - // provides extra methods. - adaptedAPI = NewSealingAPIAdapter(m.api) + // Create a shim to glue the API required by the sealing component + // with the API that Lotus is capable of providing. + // The shim translates between "tipset tokens" and tipset keys, and + // provides extra methods. + adaptedAPI := NewSealingAPIAdapter(m.api) - // Instantiate a precommit policy. - cfg = sealing.GetSealingConfigFunc(m.getSealConfig) - provingBuffer = md.WPoStProvingPeriod * 2 + // Instantiate a precommit policy. + cfg := sealing.GetSealingConfigFunc(m.getSealConfig) + provingBuffer := md.WPoStProvingPeriod * 2 - pcp = sealing.NewBasicPreCommitPolicy(adaptedAPI, cfg, provingBuffer) + pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, cfg, provingBuffer) - // address selector. - as = func(ctx context.Context, mi miner.MinerInfo, use api.AddrUse, goodFunds, minFunds abi.TokenAmount) (address.Address, abi.TokenAmount, error) { - return m.addrSel.AddressFor(ctx, m.api, mi, use, goodFunds, minFunds) - } - ) + // address selector. + as := func(ctx context.Context, mi miner.MinerInfo, use api.AddrUse, goodFunds, minFunds abi.TokenAmount) (address.Address, abi.TokenAmount, error) { + return m.addrSel.AddressFor(ctx, m.api, mi, use, goodFunds, minFunds) + } // Instantiate the sealing FSM. m.sealing = sealing.New(ctx, adaptedAPI, m.feeCfg, evtsAdapter, m.maddr, m.ds, m.sealer, m.sc, m.verif, m.prover, &pcp, cfg, m.handleSealingNotifications, as) From 82ac0a24a01b55820927371c0710ab432063893f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 9 Aug 2021 22:15:38 -0700 Subject: [PATCH 5/9] test: improve chain event tests --- chain/events/events_called.go | 2 +- chain/events/events_test.go | 448 ++++++++++++++++++++-------------- chain/events/observer.go | 2 +- 3 files changed, 269 insertions(+), 183 deletions(-) diff --git a/chain/events/events_called.go b/chain/events/events_called.go index 2b1a76e84..091b4b31a 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -393,7 +393,7 @@ type StateMatchFunc func(oldTs, newTs *types.TipSet) (bool, StateChange, error) // * `StateChangeHandler` is called when the specified state change was observed // on-chain, and a confidence threshold was reached, or the specified `timeout` // height was reached with no state change observed. When this callback is -// invoked on a timeout, `oldState` and `newState` are set to nil. +// invoked on a timeout, `oldTs` and `states are set to nil. // This callback returns a boolean specifying whether further notifications // should be sent, like `more` return param from `CheckFunc` above. // diff --git a/chain/events/events_test.go b/chain/events/events_test.go index 0f4687c8d..a0c094244 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -43,10 +43,10 @@ type fakeCS struct { tipsets map[types.TipSetKey]*types.TipSet - sub func(rev, app []*types.TipSet) - - callNumberLk sync.Mutex - callNumber map[string]int + mu sync.Mutex + waitSub chan struct{} + subCh chan<- []*api.HeadChange + callNumber map[string]int } func newFakeCS(t *testing.T) *fakeCS { @@ -58,55 +58,82 @@ func newFakeCS(t *testing.T) *fakeCS { tipsets: make(map[types.TipSetKey]*types.TipSet), tsc: newTSCache(nil, 2*build.ForkLengthThreshold), callNumber: map[string]int{}, + waitSub: make(chan struct{}), } require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) return fcs } func (fcs *fakeCS) ChainHead(ctx context.Context) (*types.TipSet, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainHead"] = fcs.callNumber["ChainHead"] + 1 panic("implement me") } func (fcs *fakeCS) ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() fcs.callNumber["ChainGetPath"] = fcs.callNumber["ChainGetPath"] + 1 - panic("Not Implemented") + fcs.mu.Unlock() + + fromTs, err := fcs.ChainGetTipSet(ctx, from) + if err != nil { + return nil, err + } + + toTs, err := fcs.ChainGetTipSet(ctx, to) + if err != nil { + return nil, err + } + + // copied from the chainstore + revert, apply, err := store.ReorgOps(func(tsk types.TipSetKey) (*types.TipSet, error) { + return fcs.ChainGetTipSet(ctx, tsk) + }, fromTs, toTs) + if err != nil { + return nil, err + } + + path := make([]*api.HeadChange, len(revert)+len(apply)) + for i, r := range revert { + path[i] = &api.HeadChange{Type: store.HCRevert, Val: r} + } + for j, i := 0, len(apply)-1; i >= 0; j, i = j+1, i-1 { + path[j+len(revert)] = &api.HeadChange{Type: store.HCApply, Val: apply[i]} + } + return path, nil } func (fcs *fakeCS) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainGetTipSet"] = fcs.callNumber["ChainGetTipSet"] + 1 return fcs.tipsets[key], nil } func (fcs *fakeCS) StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["StateSearchMsg"] = fcs.callNumber["StateSearchMsg"] + 1 return nil, nil } func (fcs *fakeCS) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["StateGetActor"] = fcs.callNumber["StateGetActor"] + 1 panic("Not Implemented") } func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainGetTipSetByHeight"] = fcs.callNumber["ChainGetTipSetByHeight"] + 1 panic("Not Implemented") } func (fcs *fakeCS) ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainGetTipSetAfterHeight"] = fcs.callNumber["ChainGetTipSetAfterHeight"] + 1 panic("Not Implemented") } @@ -158,47 +185,32 @@ func (fcs *fakeCS) makeTs(t *testing.T, parents []cid.Cid, h abi.ChainEpoch, msg } func (fcs *fakeCS) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainNotify"] = fcs.callNumber["ChainNotify"] + 1 out := make(chan []*api.HeadChange, 1) + if fcs.subCh != nil { + close(out) + fcs.t.Error("already subscribed to notifications") + return out, nil + } + best, err := fcs.tsc.ChainHead(ctx) if err != nil { return nil, err } + out <- []*api.HeadChange{{Type: store.HCCurrent, Val: best}} - - fcs.sub = func(rev, app []*types.TipSet) { - notif := make([]*api.HeadChange, len(rev)+len(app)) - - for i, r := range rev { - notif[i] = &api.HeadChange{ - Type: store.HCRevert, - Val: r, - } - } - for i, r := range app { - notif[i+len(rev)] = &api.HeadChange{ - Type: store.HCApply, - Val: r, - } - } - - select { - case out <- notif: - case <-ctx.Done(): - // TODO: fail test? - return - } - } + fcs.subCh = out + close(fcs.waitSub) return out, nil } func (fcs *fakeCS) ChainGetBlockMessages(ctx context.Context, blk cid.Cid) (*api.BlockMessages, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainGetBlockMessages"] = fcs.callNumber["ChainGetBlockMessages"] + 1 messages, ok := fcs.blkMsgs[blk] if !ok { @@ -235,11 +247,44 @@ func (fcs *fakeCS) fakeMsgs(m fakeMsg) cid.Cid { return c } -func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { // todo: allow msgs - if fcs.sub == nil { +func (fcs *fakeCS) dropSub() { + fcs.mu.Lock() + + if fcs.subCh == nil { + fcs.mu.Unlock() fcs.t.Fatal("sub not be nil") } + waitCh := make(chan struct{}) + fcs.waitSub = waitCh + close(fcs.subCh) + fcs.subCh = nil + fcs.mu.Unlock() + + <-waitCh +} + +func (fcs *fakeCS) sub(rev, app []*types.TipSet) { + <-fcs.waitSub + notif := make([]*api.HeadChange, len(rev)+len(app)) + + for i, r := range rev { + notif[i] = &api.HeadChange{ + Type: store.HCRevert, + Val: r, + } + } + for i, r := range app { + notif[i+len(rev)] = &api.HeadChange{ + Type: store.HCApply, + Val: r, + } + } + + fcs.subCh <- notif +} + +func (fcs *fakeCS) advance(rev, app, drop int, msgs map[int]cid.Cid, nulls ...int) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -255,9 +300,17 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / require.NoError(fcs.t, err) if _, ok := nullm[int(from.Height())]; !ok { - revs = append(revs, from) - require.NoError(fcs.t, fcs.tsc.revert(from)) + + if drop == 0 { + revs = append(revs, from) + } + } + if drop > 0 { + drop-- + if drop == 0 { + fcs.dropSub() + } } fcs.h-- } @@ -272,20 +325,27 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / mc = dummyCid } - if _, ok := nullm[int(fcs.h)]; ok { - continue + if _, ok := nullm[int(fcs.h)]; !ok { + best, err := fcs.tsc.ChainHead(ctx) + require.NoError(fcs.t, err) + ts := fcs.makeTs(fcs.t, best.Key().Cids(), fcs.h, mc) + require.NoError(fcs.t, fcs.tsc.add(ts)) + + if hasMsgs { + fcs.blkMsgs[ts.Blocks()[0].Cid()] = mc + } + + if drop == 0 { + apps = append(apps, ts) + } } - best, err := fcs.tsc.ChainHead(ctx) - require.NoError(fcs.t, err) - ts := fcs.makeTs(fcs.t, best.Key().Cids(), fcs.h, mc) - require.NoError(fcs.t, fcs.tsc.add(ts)) - - if hasMsgs { - fcs.blkMsgs[ts.Blocks()[0].Cid()] = mc + if drop > 0 { + drop-- + if drop == 0 { + fcs.dropSub() + } } - - apps = append(apps, ts) } fcs.sub(revs, apps) @@ -316,88 +376,47 @@ func TestAt(t *testing.T) { }, 3, 5) require.NoError(t, err) - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(10, 10, nil) + fcs.advance(10, 10, 0, nil) require.Equal(t, true, applied) require.Equal(t, true, reverted) applied = false reverted = false - fcs.advance(10, 1, nil) + fcs.advance(10, 1, 0, nil) require.Equal(t, false, applied) require.Equal(t, true, reverted) reverted = false - fcs.advance(0, 1, nil) + fcs.advance(0, 1, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 2, nil) + fcs.advance(0, 2, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 1, nil) // 8 + fcs.advance(0, 1, 0, nil) // 8 require.Equal(t, true, applied) require.Equal(t, false, reverted) } -func TestAtDoubleTrigger(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - fcs := newFakeCS(t) - - events, err := NewEvents(ctx, fcs) - require.NoError(t, err) - - var applied bool - var reverted bool - - err = events.ChainAt(ctx, func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { - require.Equal(t, 5, int(ts.Height())) - require.Equal(t, 8, int(curH)) - applied = true - return nil - }, func(_ context.Context, ts *types.TipSet) error { - reverted = true - return nil - }, 3, 5) - require.NoError(t, err) - - fcs.advance(0, 6, nil) - require.False(t, applied) - require.False(t, reverted) - - fcs.advance(0, 1, nil) - require.True(t, applied) - require.False(t, reverted) - applied = false - - fcs.advance(2, 2, nil) - require.False(t, applied) - require.False(t, reverted) - - fcs.advance(4, 4, nil) - require.True(t, applied) - require.True(t, reverted) -} - func TestAtNullTrigger(t *testing.T) { fcs := newFakeCS(t) events, err := NewEvents(context.Background(), fcs) @@ -417,11 +436,11 @@ func TestAtNullTrigger(t *testing.T) { }, 3, 5) require.NoError(t, err) - fcs.advance(0, 6, nil, 5) + fcs.advance(0, 6, 0, nil, 5) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false @@ -450,16 +469,16 @@ func TestAtNullConf(t *testing.T) { }, 3, 5) require.NoError(t, err) - fcs.advance(0, 6, nil) + fcs.advance(0, 6, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 3, nil, 8) + fcs.advance(0, 3, 0, nil, 8) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false - fcs.advance(7, 1, nil) + fcs.advance(7, 1, 0, nil) require.Equal(t, false, applied) require.Equal(t, true, reverted) reverted = false @@ -471,7 +490,7 @@ func TestAtStart(t *testing.T) { events, err := NewEvents(context.Background(), fcs) require.NoError(t, err) - fcs.advance(0, 5, nil) // 6 + fcs.advance(0, 5, 0, nil) // 6 var applied bool var reverted bool @@ -490,7 +509,7 @@ func TestAtStart(t *testing.T) { require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 5, nil) // 11 + fcs.advance(0, 5, 0, nil) // 11 require.Equal(t, true, applied) require.Equal(t, false, reverted) } @@ -501,7 +520,7 @@ func TestAtStartConfidence(t *testing.T) { events, err := NewEvents(context.Background(), fcs) require.NoError(t, err) - fcs.advance(0, 10, nil) // 11 + fcs.advance(0, 10, 0, nil) // 11 var applied bool var reverted bool @@ -545,7 +564,7 @@ func TestAtChained(t *testing.T) { }, 3, 5) require.NoError(t, err) - fcs.advance(0, 15, nil) + fcs.advance(0, 15, 0, nil) require.Equal(t, true, applied) require.Equal(t, false, reverted) @@ -557,7 +576,7 @@ func TestAtChainedConfidence(t *testing.T) { events, err := NewEvents(context.Background(), fcs) require.NoError(t, err) - fcs.advance(0, 15, nil) + fcs.advance(0, 15, 0, nil) var applied bool var reverted bool @@ -587,7 +606,7 @@ func TestAtChainedConfidenceNull(t *testing.T) { events, err := NewEvents(context.Background(), fcs) require.NoError(t, err) - fcs.advance(0, 15, nil, 5) + fcs.advance(0, 15, 0, nil, 5) var applied bool var reverted bool @@ -644,13 +663,13 @@ func TestCalled(t *testing.T) { // create few blocks to make sure nothing get's randomly called - fcs.advance(0, 4, nil) // H=5 + fcs.advance(0, 4, 0, nil) // H=5 require.Equal(t, false, applied) require.Equal(t, false, reverted) // create blocks with message (but below confidence threshold) - fcs.advance(0, 3, map[int]cid.Cid{ // msg at H=6; H=8 (confidence=2) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // msg at H=6; H=8 (confidence=2) 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -663,14 +682,14 @@ func TestCalled(t *testing.T) { // create additional block so we are above confidence threshold - fcs.advance(0, 2, nil) // H=10 (confidence=3, apply) + fcs.advance(0, 2, 0, nil) // H=10 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false // dip below confidence - fcs.advance(2, 2, nil) // H=10 (confidence=3, apply) + fcs.advance(2, 2, 0, nil) // H=10 (confidence=3, apply) require.Equal(t, false, applied) require.Equal(t, false, reverted) @@ -684,13 +703,13 @@ func TestCalled(t *testing.T) { // revert some blocks, keep the message - fcs.advance(3, 1, nil) // H=8 (confidence=1) + fcs.advance(3, 1, 0, nil) // H=8 (confidence=1) require.Equal(t, false, applied) require.Equal(t, false, reverted) // revert the message - fcs.advance(2, 1, nil) // H=7, we reverted ts with the msg execution, but not the msg itself + fcs.advance(2, 1, 0, nil) // H=7, we reverted ts with the msg execution, but not the msg itself require.Equal(t, false, applied) require.Equal(t, true, reverted) @@ -704,7 +723,7 @@ func TestCalled(t *testing.T) { }, }) - fcs.advance(0, 3, map[int]cid.Cid{ // (n2msg confidence=1) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // (n2msg confidence=1) 0: n2msg, }) @@ -713,7 +732,7 @@ func TestCalled(t *testing.T) { require.Equal(t, abi.ChainEpoch(10), appliedH) applied = false - fcs.advance(0, 2, nil) // (confidence=3) + fcs.advance(0, 2, 0, nil) // (confidence=3) require.Equal(t, true, applied) require.Equal(t, false, reverted) @@ -728,7 +747,7 @@ func TestCalled(t *testing.T) { // revert and apply at different height - fcs.advance(8, 6, map[int]cid.Cid{ // (confidence=3) + fcs.advance(8, 6, 0, map[int]cid.Cid{ // (confidence=3) 1: n2msg, }) @@ -749,7 +768,7 @@ func TestCalled(t *testing.T) { // call method again - fcs.advance(0, 5, map[int]cid.Cid{ + fcs.advance(0, 5, 0, map[int]cid.Cid{ 0: n2msg, }) @@ -758,7 +777,7 @@ func TestCalled(t *testing.T) { applied = false // send and revert below confidence, then cross confidence - fcs.advance(0, 2, map[int]cid.Cid{ + fcs.advance(0, 2, 0, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 3}, @@ -766,14 +785,14 @@ func TestCalled(t *testing.T) { }), }) - fcs.advance(2, 5, nil) // H=19, but message reverted + fcs.advance(2, 5, 0, nil) // H=19, but message reverted require.Equal(t, false, applied) require.Equal(t, false, reverted) // test timeout (it's set to 20 in the call to `events.Called` above) - fcs.advance(0, 6, nil) + fcs.advance(0, 6, 0, nil) require.Equal(t, false, applied) // not calling timeout as we received messages require.Equal(t, false, reverted) @@ -781,7 +800,7 @@ func TestCalled(t *testing.T) { // test unregistering with more more = false - fcs.advance(0, 5, map[int]cid.Cid{ + fcs.advance(0, 5, 0, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 4}, // this signals we don't want more @@ -793,7 +812,7 @@ func TestCalled(t *testing.T) { require.Equal(t, false, reverted) applied = false - fcs.advance(0, 5, map[int]cid.Cid{ + fcs.advance(0, 5, 0, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 5}, @@ -806,12 +825,12 @@ func TestCalled(t *testing.T) { // revert after disabled - fcs.advance(5, 1, nil) // try reverting msg sent after disabling + fcs.advance(5, 1, 0, nil) // try reverting msg sent after disabling require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(5, 1, nil) // try reverting msg sent before disabling + fcs.advance(5, 1, 0, nil) // try reverting msg sent before disabling require.Equal(t, false, applied) require.Equal(t, true, reverted) @@ -842,10 +861,10 @@ func TestCalledTimeout(t *testing.T) { }, 3, 20, matchAddrMethod(t0123, 5)) require.NoError(t, err) - fcs.advance(0, 21, nil) + fcs.advance(0, 21, 0, nil) require.False(t, called) - fcs.advance(0, 5, nil) + fcs.advance(0, 5, 0, nil) require.True(t, called) called = false @@ -856,9 +875,7 @@ func TestCalledTimeout(t *testing.T) { events, err = NewEvents(context.Background(), fcs) require.NoError(t, err) - // XXX: Needed to set the latest head so "check" succeeds". Is that OK? Or do we expect - // check to work _before_ we've received any events. - fcs.advance(0, 1, nil) + fcs.advance(0, 1, 0, nil) err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return true, true, nil @@ -874,10 +891,10 @@ func TestCalledTimeout(t *testing.T) { }, 3, 20, matchAddrMethod(t0123, 5)) require.NoError(t, err) - fcs.advance(0, 21, nil) + fcs.advance(0, 21, 0, nil) require.False(t, called) - fcs.advance(0, 5, nil) + fcs.advance(0, 5, 0, nil) require.False(t, called) } @@ -921,7 +938,7 @@ func TestCalledOrder(t *testing.T) { }, 3, 20, matchAddrMethod(t0123, 5)) require.NoError(t, err) - fcs.advance(0, 10, map[int]cid.Cid{ + fcs.advance(0, 10, 0, map[int]cid.Cid{ 1: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -934,7 +951,7 @@ func TestCalledOrder(t *testing.T) { }), }) - fcs.advance(9, 1, nil) + fcs.advance(9, 1, 0, nil) } func TestCalledNull(t *testing.T) { @@ -963,13 +980,13 @@ func TestCalledNull(t *testing.T) { // create few blocks to make sure nothing get's randomly called - fcs.advance(0, 4, nil) // H=5 + fcs.advance(0, 4, 0, nil) // H=5 require.Equal(t, false, applied) require.Equal(t, false, reverted) // create blocks with message (but below confidence threshold) - fcs.advance(0, 3, map[int]cid.Cid{ // msg at H=6; H=8 (confidence=2) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // msg at H=6; H=8 (confidence=2) 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -983,13 +1000,13 @@ func TestCalledNull(t *testing.T) { // create additional blocks so we are above confidence threshold, but with null tipset at the height // of application - fcs.advance(0, 3, nil, 10) // H=11 (confidence=3, apply) + fcs.advance(0, 3, 0, nil, 10) // H=11 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false - fcs.advance(5, 1, nil, 10) + fcs.advance(5, 1, 0, nil, 10) require.Equal(t, false, applied) require.Equal(t, true, reverted) @@ -1021,13 +1038,13 @@ func TestRemoveTriggersOnMessage(t *testing.T) { // create few blocks to make sure nothing get's randomly called - fcs.advance(0, 4, nil) // H=5 + fcs.advance(0, 4, 0, nil) // H=5 require.Equal(t, false, applied) require.Equal(t, false, reverted) // create blocks with message (but below confidence threshold) - fcs.advance(0, 3, map[int]cid.Cid{ // msg occurs at H=5, applied at H=6; H=8 (confidence=2) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // msg occurs at H=5, applied at H=6; H=8 (confidence=2) 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -1039,19 +1056,19 @@ func TestRemoveTriggersOnMessage(t *testing.T) { require.Equal(t, false, reverted) // revert applied TS & message TS - fcs.advance(3, 1, nil) // H=6 (tipset message applied in reverted, AND message reverted) + fcs.advance(3, 1, 0, nil) // H=6 (tipset message applied in reverted, AND message reverted) require.Equal(t, false, applied) require.Equal(t, false, reverted) // create additional blocks so we are above confidence threshold, but message not applied // as it was reverted - fcs.advance(0, 5, nil) // H=11 (confidence=3, apply) + fcs.advance(0, 5, 0, nil) // H=11 (confidence=3, apply) require.Equal(t, false, applied) require.Equal(t, false, reverted) // create blocks with message again (but below confidence threshold) - fcs.advance(0, 3, map[int]cid.Cid{ // msg occurs at H=12, applied at H=13; H=15 (confidence=2) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // msg occurs at H=12, applied at H=13; H=15 (confidence=2) 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 2}, @@ -1062,12 +1079,12 @@ func TestRemoveTriggersOnMessage(t *testing.T) { require.Equal(t, false, reverted) // revert applied height TS, but don't remove message trigger - fcs.advance(2, 1, nil) // H=13 (tipset message applied in reverted, by tipset with message not reverted) + fcs.advance(2, 1, 0, nil) // H=13 (tipset message applied in reverted, by tipset with message not reverted) require.Equal(t, false, applied) require.Equal(t, false, reverted) // create additional blocks so we are above confidence threshold - fcs.advance(0, 4, nil) // H=18 (confidence=3, apply) + fcs.advance(0, 4, 0, nil) // H=18 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) @@ -1098,6 +1115,9 @@ func TestStateChanged(t *testing.T) { err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { + if data != nil { + require.Equal(t, oldTs.Key(), newTs.Parents()) + } require.Equal(t, false, applied) applied = true appliedData = data @@ -1109,6 +1129,7 @@ func TestStateChanged(t *testing.T) { reverted = true return nil }, confidence, timeout, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { + require.Equal(t, oldTs.Key(), newTs.Parents()) if matchData == nil { return false, matchData, nil } @@ -1121,27 +1142,27 @@ func TestStateChanged(t *testing.T) { // create few blocks to make sure nothing get's randomly called - fcs.advance(0, 4, nil) // H=5 + fcs.advance(0, 4, 0, nil) // H=5 require.Equal(t, false, applied) require.Equal(t, false, reverted) // create state change (but below confidence threshold) matchData = testStateChange{from: "a", to: "b"} - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) // create additional block so we are above confidence threshold - fcs.advance(0, 2, nil) // H=10 (confidence=3, apply) + fcs.advance(0, 2, 0, nil) // H=10 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false // dip below confidence (should not apply again) - fcs.advance(2, 2, nil) // H=10 (confidence=3, apply) + fcs.advance(2, 2, 0, nil) // H=10 (confidence=3, apply) require.Equal(t, false, applied) require.Equal(t, false, reverted) @@ -1175,6 +1196,9 @@ func TestStateChangedRevert(t *testing.T) { err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { + if data != nil { + require.Equal(t, oldTs.Key(), newTs.Parents()) + } require.Equal(t, false, applied) applied = true return more, nil @@ -1182,6 +1206,8 @@ func TestStateChangedRevert(t *testing.T) { reverted = true return nil }, confidence, timeout, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { + require.Equal(t, oldTs.Key(), newTs.Parents()) + if matchData == nil { return false, matchData, nil } @@ -1192,18 +1218,18 @@ func TestStateChangedRevert(t *testing.T) { }) require.NoError(t, err) - fcs.advance(0, 2, nil) // H=3 + fcs.advance(0, 2, 0, nil) // H=3 // Make a state change from TS at height 3 to TS at height 4 matchData = testStateChange{from: "a", to: "b"} - fcs.advance(0, 1, nil) // H=4 + fcs.advance(0, 1, 0, nil) // H=4 // Haven't yet reached confidence require.Equal(t, false, applied) require.Equal(t, false, reverted) // Advance to reach confidence level - fcs.advance(0, 1, nil) // H=5 + fcs.advance(0, 1, 0, nil) // H=5 // Should now have called the handler require.Equal(t, true, applied) @@ -1211,19 +1237,19 @@ func TestStateChangedRevert(t *testing.T) { applied = false // Advance 3 more TS - fcs.advance(0, 3, nil) // H=8 + fcs.advance(0, 3, 0, nil) // H=8 require.Equal(t, false, applied) require.Equal(t, false, reverted) // Regress but not so far as to cause a revert - fcs.advance(3, 1, nil) // H=6 + fcs.advance(3, 1, 0, nil) // H=6 require.Equal(t, false, applied) require.Equal(t, false, reverted) // Regress back to state where change happened - fcs.advance(3, 1, nil) // H=4 + fcs.advance(3, 1, 0, nil) // H=4 // Expect revert to have happened require.Equal(t, false, applied) @@ -1241,6 +1267,9 @@ func TestStateChangedTimeout(t *testing.T) { err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { + if data != nil { + require.Equal(t, oldTs.Key(), newTs.Parents()) + } called = true require.Nil(t, data) require.Equal(t, abi.ChainEpoch(20), newTs.Height()) @@ -1250,15 +1279,16 @@ func TestStateChangedTimeout(t *testing.T) { t.Fatal("revert on timeout") return nil }, 3, 20, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { + require.Equal(t, oldTs.Key(), newTs.Parents()) return false, nil, nil }) require.NoError(t, err) - fcs.advance(0, 21, nil) + fcs.advance(0, 21, 0, nil) require.False(t, called) - fcs.advance(0, 5, nil) + fcs.advance(0, 5, 0, nil) require.True(t, called) called = false @@ -1268,13 +1298,14 @@ func TestStateChangedTimeout(t *testing.T) { events, err = NewEvents(context.Background(), fcs) require.NoError(t, err) - // XXX: Needed to set the latest head so "check" succeeds". Is that OK? Or do we expect - // check to work _before_ we've received any events. - fcs.advance(0, 1, nil) + fcs.advance(0, 1, 0, nil) err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return true, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { + if data != nil { + require.Equal(t, oldTs.Key(), newTs.Parents()) + } called = true require.Nil(t, data) require.Equal(t, abi.ChainEpoch(20), newTs.Height()) @@ -1284,14 +1315,15 @@ func TestStateChangedTimeout(t *testing.T) { t.Fatal("revert on timeout") return nil }, 3, 20, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { + require.Equal(t, oldTs.Key(), newTs.Parents()) return false, nil, nil }) require.NoError(t, err) - fcs.advance(0, 21, nil) + fcs.advance(0, 21, 0, nil) require.False(t, called) - fcs.advance(0, 5, nil) + fcs.advance(0, 5, 0, nil) require.False(t, called) } @@ -1335,7 +1367,7 @@ func TestCalledMultiplePerEpoch(t *testing.T) { }, 3, 20, matchAddrMethod(t0123, 5)) require.NoError(t, err) - fcs.advance(0, 10, map[int]cid.Cid{ + fcs.advance(0, 10, 0, map[int]cid.Cid{ 1: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -1344,7 +1376,7 @@ func TestCalledMultiplePerEpoch(t *testing.T) { }), }) - fcs.advance(9, 1, nil) + fcs.advance(9, 1, 0, nil) } func TestCachedSameBlock(t *testing.T) { @@ -1353,9 +1385,63 @@ func TestCachedSameBlock(t *testing.T) { _, err := NewEvents(context.Background(), fcs) require.NoError(t, err) - fcs.advance(0, 10, map[int]cid.Cid{}) + fcs.advance(0, 10, 0, map[int]cid.Cid{}) assert.Assert(t, fcs.callNumber["ChainGetBlockMessages"] == 20, "expect call ChainGetBlockMessages %d but got ", 20, fcs.callNumber["ChainGetBlockMessages"]) - fcs.advance(5, 10, map[int]cid.Cid{}) + fcs.advance(5, 10, 0, map[int]cid.Cid{}) assert.Assert(t, fcs.callNumber["ChainGetBlockMessages"] == 30, "expect call ChainGetBlockMessages %d but got ", 30, fcs.callNumber["ChainGetBlockMessages"]) } + +type testObserver struct { + t *testing.T + head *types.TipSet +} + +func (t *testObserver) Apply(_ context.Context, from, to *types.TipSet) error { + if t.head != nil { + require.True(t.t, t.head.Equals(from)) + } + t.head = to + return nil +} + +func (t *testObserver) Revert(_ context.Context, from, to *types.TipSet) error { + if t.head != nil { + require.True(t.t, t.head.Equals(from)) + } + t.head = to + return nil +} + +func TestReconnect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fcs := newFakeCS(t) + + events, err := NewEvents(ctx, fcs) + require.NoError(t, err) + + fcs.advance(0, 1, 0, nil) + + events.Observe(&testObserver{t: t}) + + fcs.advance(0, 3, 0, nil) + + // Drop on apply + fcs.advance(0, 6, 2, nil) + require.True(t, fcs.callNumber["ChainGetPath"] == 1) + + // drop across revert/apply boundary + fcs.advance(4, 2, 3, nil) + require.True(t, fcs.callNumber["ChainGetPath"] == 2) + fcs.advance(0, 6, 0, nil) + + // drop on revert + fcs.advance(3, 0, 2, nil) + require.True(t, fcs.callNumber["ChainGetPath"] == 3) + + // drop with nulls + fcs.advance(0, 5, 2, nil, 0, 1, 3) + require.True(t, fcs.callNumber["ChainGetPath"] == 4) +} diff --git a/chain/events/observer.go b/chain/events/observer.go index cd25b4874..52fc1de25 100644 --- a/chain/events/observer.go +++ b/chain/events/observer.go @@ -111,7 +111,7 @@ func (o *observer) listenHeadChangesOnce(ctx context.Context) error { } if err := o.applyChanges(ctx, changes); err != nil { - return xerrors.Errorf("failed to apply head changes: %w", err) + return xerrors.Errorf("failed catch-up head changes: %w", err) } } From f518e34131a54910df3ebcd93455d5a012802cc7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 10 Aug 2021 23:53:57 -0700 Subject: [PATCH 6/9] fix: atomically get head when registering an observer This lets us always call check (accurately). --- chain/events/events.go | 10 ++---- chain/events/events_called.go | 25 ++++++-------- chain/events/events_height.go | 21 ++++-------- chain/events/events_test.go | 4 --- chain/events/observer.go | 63 +++++++++++++++++++++++------------ 5 files changed, 61 insertions(+), 62 deletions(-) diff --git a/chain/events/events.go b/chain/events/events.go index 1e39d3646..5c494fcb0 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -50,17 +50,13 @@ func NewEventsWithConfidence(ctx context.Context, api EventAPI, gcConfidence abi cache := newCache(api, gcConfidence) ob := newObserver(cache, gcConfidence) - he := newHeightEvents(cache, gcConfidence) - headChange := newHCEvents(cache) - - // Cache first. Observers are ordered and we always want to fill the cache first. - ob.Observe(cache.observer()) - ob.Observe(he.observer()) - ob.Observe(headChange.observer()) if err := ob.start(ctx); err != nil { return nil, err } + he := newHeightEvents(cache, ob, gcConfidence) + headChange := newHCEvents(cache, ob) + return &Events{ob, he, headChange}, nil } diff --git a/chain/events/events_called.go b/chain/events/events_called.go index 091b4b31a..026ad8a4e 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -69,10 +69,9 @@ type queuedEvent struct { type hcEvents struct { cs EventAPI + lk sync.Mutex lastTs *types.TipSet - lk sync.Mutex - ctr triggerID // TODO: get rid of trigger IDs and just use pointers as keys. @@ -93,7 +92,7 @@ type hcEvents struct { watcherEvents } -func newHCEvents(api EventAPI) *hcEvents { +func newHCEvents(api EventAPI, obs *observer) *hcEvents { e := &hcEvents{ cs: api, confQueue: map[triggerH]map[msgH][]*queuedEvent{}, @@ -105,15 +104,16 @@ func newHCEvents(api EventAPI) *hcEvents { e.messageEvents = newMessageEvents(e, api) e.watcherEvents = newWatcherEvents(e, api) + // We need to take the lock as the observer could immediately try calling us. + e.lk.Lock() + e.lastTs = obs.Observe((*hcEventsObserver)(e)) + e.lk.Unlock() + return e } type hcEventsObserver hcEvents -func (e *hcEvents) observer() TipSetObserver { - return (*hcEventsObserver)(e) -} - func (e *hcEventsObserver) Apply(ctx context.Context, from, to *types.TipSet) error { e.lk.Lock() defer e.lk.Unlock() @@ -284,14 +284,9 @@ func (e *hcEvents) onHeadChanged(ctx context.Context, check CheckFunc, hnd Event defer e.lk.Unlock() // Check if the event has already occurred - more := true - done := false - if e.lastTs != nil { - var err error - done, more, err = check(ctx, e.lastTs) - if err != nil { - return 0, xerrors.Errorf("called check error (h: %d): %w", e.lastTs.Height(), err) - } + done, more, err := check(ctx, e.lastTs) + if err != nil { + return 0, xerrors.Errorf("called check error (h: %d): %w", e.lastTs.Height(), err) } if done { timeout = NoTimeout diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 02c252bc9..73df04be6 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -30,13 +30,17 @@ type heightEvents struct { lastGc abi.ChainEpoch //nolint:structcheck } -func newHeightEvents(api EventAPI, gcConfidence abi.ChainEpoch) *heightEvents { - return &heightEvents{ +func newHeightEvents(api EventAPI, obs *observer, gcConfidence abi.ChainEpoch) *heightEvents { + he := &heightEvents{ api: api, gcConfidence: gcConfidence, tsHeights: map[abi.ChainEpoch][]*heightHandler{}, triggerHeights: map[abi.ChainEpoch][]*heightHandler{}, } + he.lk.Lock() + he.head = obs.Observe((*heightEventsObserver)(he)) + he.lk.Unlock() + return he } // ChainAt invokes the specified `HeightHandler` when the chain reaches the @@ -69,15 +73,6 @@ func (e *heightEvents) ChainAt(ctx context.Context, hnd HeightHandler, rev Rever e.lk.Lock() for { head := e.head - - // If we haven't initialized yet, store the trigger and move on. - if head == nil { - e.triggerHeights[triggerAt] = append(e.triggerHeights[triggerAt], handler) - e.tsHeights[h] = append(e.tsHeights[h], handler) - e.lk.Unlock() - return nil - } - if head.Height() >= h { // Head is past the handler height. We at least need to stash the tipset to // avoid doing this from the main event loop. @@ -152,10 +147,6 @@ func (e *heightEvents) ChainAt(ctx context.Context, hnd HeightHandler, rev Rever } } -func (e *heightEvents) observer() TipSetObserver { - return (*heightEventsObserver)(e) -} - // Updates the head and garbage collects if we're 2x over our garbage collection confidence period. func (e *heightEventsObserver) updateHead(h *types.TipSet) { e.lk.Lock() diff --git a/chain/events/events_test.go b/chain/events/events_test.go index a0c094244..0536b5ebb 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -875,8 +875,6 @@ func TestCalledTimeout(t *testing.T) { events, err = NewEvents(context.Background(), fcs) require.NoError(t, err) - fcs.advance(0, 1, 0, nil) - err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return true, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { @@ -1298,8 +1296,6 @@ func TestStateChangedTimeout(t *testing.T) { events, err = NewEvents(context.Background(), fcs) require.NoError(t, err) - fcs.advance(0, 1, 0, nil) - err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return true, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { diff --git a/chain/events/observer.go b/chain/events/observer.go index 52fc1de25..c67d821b5 100644 --- a/chain/events/observer.go +++ b/chain/events/observer.go @@ -18,25 +18,26 @@ import ( type observer struct { api EventAPI - lk sync.Mutex gcConfidence abi.ChainEpoch ready chan struct{} + lk sync.Mutex head *types.TipSet maxHeight abi.ChainEpoch - observers []TipSetObserver } -func newObserver(api EventAPI, gcConfidence abi.ChainEpoch) *observer { - return &observer{ +func newObserver(api *cache, gcConfidence abi.ChainEpoch) *observer { + obs := &observer{ api: api, gcConfidence: gcConfidence, ready: make(chan struct{}), observers: []TipSetObserver{}, } + obs.Observe(api.observer()) + return obs } func (o *observer) start(ctx context.Context) error { @@ -100,12 +101,18 @@ func (o *observer) listenHeadChangesOnce(ctx context.Context) error { return xerrors.Errorf("expected first head notification type to be 'current', was '%s'", cur[0].Type) } - head := cur[0].Val + curHead := cur[0].Val + + o.lk.Lock() if o.head == nil { - o.head = head + o.head = curHead close(o.ready) - } else if !o.head.Equals(head) { - changes, err := o.api.ChainGetPath(ctx, o.head.Key(), head.Key()) + } + startHead := o.head + o.lk.Unlock() + + if !startHead.Equals(curHead) { + changes, err := o.api.ChainGetPath(ctx, startHead.Key(), curHead.Key()) if err != nil { return xerrors.Errorf("failed to get path from last applied tipset to head: %w", err) } @@ -152,18 +159,23 @@ func (o *observer) headChange(ctx context.Context, rev, app []*types.TipSet) err ctx, span := trace.StartSpan(ctx, "events.HeadChange") span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev)))) span.AddAttributes(trace.Int64Attribute("applies", int64(len(app)))) + + o.lk.Lock() + head := o.head + o.lk.Unlock() + defer func() { - span.AddAttributes(trace.Int64Attribute("endHeight", int64(o.head.Height()))) + span.AddAttributes(trace.Int64Attribute("endHeight", int64(head.Height()))) span.End() }() // NOTE: bailing out here if the head isn't what we expected is fine. We'll re-start the // entire process and handle any strange reorgs. for i, from := range rev { - if !from.Equals(o.head) { + if !from.Equals(head) { return xerrors.Errorf( "expected to revert %s (%d), reverting %s (%d)", - o.head.Key(), o.head.Height(), from.Key(), from.Height(), + head.Key(), head.Height(), from.Key(), from.Height(), ) } var to *types.TipSet @@ -171,7 +183,7 @@ func (o *observer) headChange(ctx context.Context, rev, app []*types.TipSet) err // If we have more reverts, the next revert is the next head. to = rev[i+1] } else { - // At the end of the revert sequenece, we need to looup the joint tipset + // At the end of the revert sequenece, we need to lookup the joint tipset // between the revert sequence and the apply sequence. var err error to, err = o.api.ChainGetTipSet(ctx, from.Parents()) @@ -181,9 +193,14 @@ func (o *observer) headChange(ctx context.Context, rev, app []*types.TipSet) err } } - // Get the observers late in case an observer registers/unregisters itself. + // Get the current observers and atomically set the head. + // + // 1. We need to get the observers every time in case some registered/deregistered. + // 2. We need to atomically set the head so new observers don't see events twice or + // skip them. o.lk.Lock() observers := o.observers + o.head = to o.lk.Unlock() for _, obs := range observers { @@ -196,39 +213,43 @@ func (o *observer) headChange(ctx context.Context, rev, app []*types.TipSet) err log.Errorf("reverted past finality, from %d to %d", o.maxHeight, to.Height()) } - o.head = to + head = to } for _, to := range app { - if to.Parents() != o.head.Key() { + if to.Parents() != head.Key() { return xerrors.Errorf( "cannot apply %s (%d) with parents %s on top of %s (%d)", - to.Key(), to.Height(), to.Parents(), o.head.Key(), o.head.Height(), + to.Key(), to.Height(), to.Parents(), head.Key(), head.Height(), ) } - // Get the observers late in case an observer registers/unregisters itself. o.lk.Lock() observers := o.observers + o.head = to o.lk.Unlock() for _, obs := range observers { - if err := obs.Apply(ctx, o.head, to); err != nil { + if err := obs.Apply(ctx, head, to); err != nil { log.Errorf("observer %T failed to revert tipset %s (%d) with: %s", obs, to.Key(), to.Height(), err) } } - o.head = to if to.Height() > o.maxHeight { o.maxHeight = to.Height() } + head = to } return nil } -// TODO: add a confidence level so we can have observers with difference levels of confidence -func (o *observer) Observe(obs TipSetObserver) { +// Observe registers the observer, and returns the current tipset. The observer is guaranteed to +// observe events starting at this tipset. +// +// Returns nil if the observer hasn't started yet (but still registers). +func (o *observer) Observe(obs TipSetObserver) *types.TipSet { o.lk.Lock() defer o.lk.Unlock() o.observers = append(o.observers, obs) + return o.head } From 003eae81ced7bbf78bb9b1d4454962caed099837 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 27 Aug 2021 12:12:55 -0700 Subject: [PATCH 7/9] fix: address review --- chain/events/events_height.go | 2 +- itests/api_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 73df04be6..6734d3ca0 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -47,7 +47,7 @@ func newHeightEvents(api EventAPI, obs *observer, gcConfidence abi.ChainEpoch) * // specified height+confidence threshold. If the chain is rolled-back under the // specified height, `RevertHandler` will be called. // -// ts passed to handlers is the tipset at the specified, or above, if lower tipsets were null +// ts passed to handlers is the tipset at the specified epoch, or above if lower tipsets were null. // // The context governs cancellations of this call, it won't cancel the event handler. func (e *heightEvents) ChainAt(ctx context.Context, hnd HeightHandler, rev RevertHandler, confidence int, h abi.ChainEpoch) error { diff --git a/itests/api_test.go b/itests/api_test.go index 9a21c9dfc..c380a6ed8 100644 --- a/itests/api_test.go +++ b/itests/api_test.go @@ -195,7 +195,7 @@ func (ts *apiSuite) testSlowNotify(t *testing.T) { full.WaitTillChain(ctx, kit.HeightAtLeast(baseHeight+100)) - // Make sure they were all closed. + // Make sure they were all closed, draining any buffered events first. for _, ch := range newHeadsChans { var ok bool for ok { From 1cf556c3a28dd62951f0727e92c288466a0641b0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 27 Aug 2021 12:25:43 -0700 Subject: [PATCH 8/9] feat: expose ChainGetPath on the gateway --- api/api_gateway.go | 1 + api/proxy_gen.go | 13 +++++++++++++ build/openrpc/full.json.gz | Bin 25415 -> 25416 bytes build/openrpc/miner.json.gz | Bin 10374 -> 10378 bytes build/openrpc/worker.json.gz | Bin 2709 -> 2710 bytes build/params_2k.go | 1 + build/params_butterfly.go | 1 + build/params_calibnet.go | 1 + build/params_debug.go | 1 + build/params_interop.go | 1 + build/params_mainnet.go | 9 ++------- build/params_nerpanet.go | 1 + build/params_shared_vals.go | 1 + build/params_testground.go | 1 + build/tools.go | 3 ++- chain/types/message_fuzz.go | 3 ++- cmd/lotus/daemon.go | 1 + cmd/lotus/daemon_nodaemon.go | 1 + cmd/lotus/debug_advance.go | 1 + .../sector-storage/ffiwrapper/prover_cgo.go | 3 ++- .../sector-storage/ffiwrapper/sealer_cgo.go | 3 ++- .../sector-storage/ffiwrapper/verifier_cgo.go | 3 ++- extern/sector-storage/fsutil/dealloc_other.go | 1 + gateway/node.go | 5 +++++ lib/ulimit/ulimit_freebsd.go | 1 + lib/ulimit/ulimit_test.go | 1 + lib/ulimit/ulimit_unix.go | 1 + 27 files changed, 46 insertions(+), 12 deletions(-) diff --git a/api/api_gateway.go b/api/api_gateway.go index 29cd8ce24..862c6ddb5 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -33,6 +33,7 @@ type Gateway interface { ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) + ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*HeadChange, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetAfterHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) diff --git a/api/proxy_gen.go b/api/proxy_gen.go index f2dc7c560..c03b83531 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -480,6 +480,8 @@ type GatewayStruct struct { ChainGetMessage func(p0 context.Context, p1 cid.Cid) (*types.Message, error) `` + ChainGetPath func(p0 context.Context, p1 types.TipSetKey, p2 types.TipSetKey) ([]*HeadChange, error) `` + ChainGetTipSet func(p0 context.Context, p1 types.TipSetKey) (*types.TipSet, error) `` ChainGetTipSetAfterHeight func(p0 context.Context, p1 abi.ChainEpoch, p2 types.TipSetKey) (*types.TipSet, error) `` @@ -3039,6 +3041,17 @@ func (s *GatewayStub) ChainGetMessage(p0 context.Context, p1 cid.Cid) (*types.Me return nil, ErrNotSupported } +func (s *GatewayStruct) ChainGetPath(p0 context.Context, p1 types.TipSetKey, p2 types.TipSetKey) ([]*HeadChange, error) { + if s.Internal.ChainGetPath == nil { + return *new([]*HeadChange), ErrNotSupported + } + return s.Internal.ChainGetPath(p0, p1, p2) +} + +func (s *GatewayStub) ChainGetPath(p0 context.Context, p1 types.TipSetKey, p2 types.TipSetKey) ([]*HeadChange, error) { + return *new([]*HeadChange), ErrNotSupported +} + func (s *GatewayStruct) ChainGetTipSet(p0 context.Context, p1 types.TipSetKey) (*types.TipSet, error) { if s.Internal.ChainGetTipSet == nil { return nil, ErrNotSupported diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 4892e090148247a49af2cd354ba5a370f055ae76..610a0591a8706d1b00af4e488208b8522eee0011 100644 GIT binary patch delta 15686 zcma*OLtvm?)3qDhHafQ5vDvY0+qz?SY}>YNqhs4fhn@4h-{>En(XKtIX;ocot+7$i z@lnwFXE0zh%UI5gh;sEZaE&Hr35F1!b3HXdtyyVcCO3$R4aZ+?_W&Pj%M^kC=}Qv* zqtdc7Gn}97thDs2%r0H$S+lZ32h#~Q+Qc=iABOb`l+?Qr7b6LFFJe^pzOPxI?w?8v zP!+EC9@8>n`@%xt75r6mP1v9%IzD2XbJBV|$_CtjFMnv-Vf73>xqCbE*;-T?g%c_y zc%(3-*y~i!OC1v>=~TE=VQQD}$S`opW>ki4E0dFmC9Dx32!Z7aaZ>HD7lw>p(WL*t zxudxgLllZcyD9u_fE)g^FK}29-Fv{2acXo}Qi*xY6s%rzj7_Y4D+FKGq5YQ=ifV-2 z+dP2sSmBElg(T2bBf(nNsnI{Rj=`344^Fmx10f#6b(JSs73QScS#g#<6SiCRC z24K1GFk+T5Kg7Ci#c5rAJbnM*AN)~i!m*xwBPSsEh;aNlidp}N@|q zBdB2}VT|q7iFo#bYK-s(QdVOZV8Vz}wFXRE_oI9ks8*5&?8U#2ga5KGUCRQK^QBjB zKQqCCCh$49enj!s82{yi+I313Fbs|2i&0V}{#% z4QXrru!zt-)O_FI>E)I=cS-AY)sj=<@qYFO=XUt9hg+g+bF~Z2pHfE+eB?&)eF1O^ z!{$TG-se9^T=g(PkiHJ}B~^#4hh}3f>^j=ciVvKsGFUcffj4p-0CMJ)O?LMZ+kVt<-zX;;jTIk+1tWos@Q zafQT!=&sE~wXErQ7tSooI6$tm;ZYYnx+_^6`9xRk#%k?t?tY%Wn#n-(ER*J2N9I&c zyJ4Q+<+$c~wpir*y-c{XgeY?{@vNxnD3HDG9fa<(svtYc8b9qQ(YKJ`I{2`G8if>u ztT?Qu(C~NGQ93uEEiebbnHCVhw91I1~28cmv1odJm zGXkp7kgSMw{-1l6Sj`Pe&mSM^nIDKJDk$^#!`yp`@B;GChf0I>aX;=+A&<|nyc#r9 zouLE`y0@XRiDTym#(*|#Vgcg7N8~&}kOwSJFy|AWpc}F;ua6(>Rv<6%uLBcftY8l` z@osKjpV#*b0RaH!(EoP6IdteT7QVN4^tkU(I8rpXw{!}x#I$-fVzCgU^>KIfx2R3B zG9&tM>9Gwv?Z_i+u_iug#ug?yAfae6%x7521XCgI&Hp)PD0l&sR!1FBi6P7j(;>L7%PuT_ns^ z=boG2Ws$4ij`{OtVNdP9n5_}YnBs33VFTmrCy#}1N}$PrfQY!{B|!E^&m;oC>hb5F z)=9ZdulQaecn8W@_8Y*t|y^B>r7v zc5j;nBNH<`+lF@&KKOfTP`7d#BzeHrfpR5L!6|{cXC`^Waf5C)HNoM5fwncxBKU&- z_mf4k0{@BamTO4!w^G;fWm-qI&1CCES0`DwzahcXywYR|p{V~j zLPsk8-?6CM5GMikmo|;2JJe|iz@zToJkN9k21+)PMX68SSBuZFM4n@cc3WQqw8Tn} zJRO5HQjwO=9+u^Jik0oGi2SDO&to8!j+OwZ2Z(hos6{(NdEKOXT2;baReoQ@(>C#8OG1EFHU;B@ zK>`|CGub3MwK+V(E5mcjFLHS5g*TW`R#kV$XN%K6_6ZkFg!LJKVqfE&3b&2ir5B>g z*1w&UsS(Cn2F!WysNLt(h=~`1vS{RAbz*Qh!A&V3H2hre$WX>;iXF)<1X77eJm7oU zDwkqG=&;DD%O~uot@Ox~(xCew3%} zLK`si6tgJJLcm)SlAJ01hzrs}ijbRCMRJqL#xg-b8 ze4bvio*`__50FVcF!K{orwhe@mW$7Kn&WFI|!~! zPm6e0HH_L^0=E7!acgLyc@Y#-N1lvtmsGJhZnE7SC%UF4^CeTPY|K$F9mz>p7xIz; z8rWZHTv|su{HJ&r{3?(RC^qZ1tTd5$nyPE{9+Cc*m}On8em%U`#93DhoUd|<{;y>? zqDG1Msl`0-pK;zD0`-@a08^WF;i$~$KO$)&Ke^H7P0-*4p(>YC)E(|@5xL@&rkjU1 zN&4o8;;&F>R49foLa`(0_SdU7aItr?0>n9FaZA)}=ksT9 z;s-9cl$2@`6sK$J+g|tY$Fo&AJsKitMVE2~S@2-Z?k$_P9_8#*J&1P@abQoe64vcG z57|GNA)Abqt(}94^Ew&8=IWwe-h^dr_VneNJId`8EjpDtYsy_Rpf~UiqYlPz<45>N zRqwxg)e8o&Zm3~~fK0HF>&)>UVo{l3@bR8=tLf5fBjZ|cLaEPp`8=ZRLs;S774iSp zVS~PPZjzICOGhC&9h=i2O#+djucn_AWcS!GCY&ji2uZ=<66M{vnUQ#3kiAv8QgjuS zP5YKucguvMT|=H8IJ+;*J19_L7X`)g9w<2vHBn!^^EJ&@K(wGx+ESvA<=Aa~t+RI- zWr)50)1OdEYpPl|I`|TI6-!~7;i{0t9SA-kdV+SQ3c{igoo|3}ynCHXu=BmScgY9v)C{a^rqrj?3QF@(qZ_hMS ztYcvNToQJn6umJN81DcPLI9OMn-FXO9@IR$l?ea}%+gYB!MaiXxlL3228UXkqb%JM zX>`Wb9^t3u)Ru2R@~K&>MQ`kG84E?pcaOfd!@y0-Yq2z%et={Z=@&m%5o`$c5xRY39aIye>aC{pG@u{>qcf_fuvd}-Q^ zAcX?~kf8*x!^;Sn;XWzWCVL;7`@;k1I$hJz}$u_Qv z)On2PPX)Ia)rlM9tCpi#^bGjI{aEftK)s3G&(VKke)>^%%+2%Je|&e@Iewwc4gr%| zGwmLvp<0vFY&@IwKv|4PpTjgjZk9lhOz3?8d~5xFCjsAzYV3*RWvI{7!T)Dl^(n5H(@lQnpN|-ktEF#mJ^*PZ}bpb9MU9E@3>(FaEiD z!)Ol{f(Y?6AU1E3A;0OfyoMx z;vo!(?@W2ub~koda&zYKzof_xgWvcWapAzAsawMrSu&QigD090BN znh`$(&HVaN?vqZ3&echJvo$M-P`!W$u*a@d-Sl4#2DE^rMv9x^*Zbw9+k)AW3lC31w9;rK63m8 zFv|G&s6s2pl{C{qg|eLS{n4)Hb~$w1Ahoc_d0oAW}->(Rpi zk>0k=Z9RE6FX$cX^ELXbvCQ&e1H>Q@Hl(xm19iCHAe8zj(Z4~M69j4&XWTFpZzp-0 zYt{;C7FbKM%a%>;>>RFK-OSze-H2>i2 z@izInth&t>tQXus_!tVUF$Aa7ZRX|G3y`vsOAmUaArLz1&6ks$8Lg@={nOgjns@J< zt_0u4=L# zKBlsqpImUj4$QIq{ehn&u7mxJ=c3TS9XX;jWo}`Ce-$GIEC#8RHGZ^*3==}R9iwxUko5!am4*O8t=cZc6WNve>_weEb@*w%0Ngl8G|zI;a$ z?x}bd73a&!O9H)5{1UHqZ5PH!G`3zak>QiPaTNU;EgCVmY5fmTy#fWkya%-8yAcN* z47r~cgv{NA#qdCg2g!DzEA*lU%p&2HTm9;QKC}EtGG~uqyN`ZZQ&iNlZcEg>p6nHYMEXttTE%5Hiudo#LZlTl~NrwHp;l_kfdN$Tm zXUqG-;n0LcuAcMz1GS!xWsKDoAoNq7Jk1_XQZqReAZq~Z-{*K%xx^cM|F9V{8g!l3+)_c)BEZD^ex{(gHTZl1lPfFy7Q zWK{0!m66gJ)9(?6H;^uf3I>h^;d1&PBg z3;b-eCN<#n7p=sPuzHj>3XIke2T`_CooNU;d?f*fen!7GH(Fk)X{R7rlGq=mBI7io zdlX9i1BT$bI5bRWn&?o+13WmSYSAhIFwtZXC+zB==T9Jvu$`eq$R~|lNd!uXzgRjF zfdLr1r)X~3vg7{IT@DcC+IouS9&x!D=+Ck{j;W-E6H z*q1Y_7@^n9lH|-XL87~qx48PeTWPCrm;)R>pE4H^6nxD)cRuCQF{Q7j>x9KTXm2gM zp}IPcs6ge$Td^daqwtR6wutR^KZKi0zdygVljRkNq&?47RYzND*R3nEd$qpMu`tE6 zzyw^nO+jg#dZ#!)zkH`$=(_FvQ8;#{UG>X2Y@BR~m6`VITk<4cvD12XmWfd@&PF>- z#wiqrff*Q z{zbz)LEntrUPyC)BxnVqQQ)-IPa?FX{0or&8ezkBF5$*#Ms!mr%Q^<1O61>LTq&M4Dm0*Oi^dsS@% z(ktOd)>feLg4@d3#@VLs?6Oor@>NA*!&Zj0#}{x5(hL5IjJnrX4X-?Rcf^0-Pz#VE z?NJLs!*#idd|+NT^Yrb#48B_+ECz9)#yKN}&Oy(&H@*4ymbwBRU-K-w9zYVjBkhTrn&`L^ZZzcSG zP3udAjKo5U%f-bM%XHmtZsZAL<0Bve-&)~gukfy9*$O{5EnzuAi{6NaltKQ%jHIGA zh&;ec^@@jj2-VjZYa1{Zgg$gG$h;Wb+mTf;D$B?Tq1pIdsX53zE-$yEg)vD3RO{0UpO^Kq0 zrw5>jr@wz~(Hap*5ah>uFMYy``93xz#VYV_7ZA7oT%mngR`+9> zXk4g|Vy|N?S?-(+z7E5Ani4y>qCv127^vWN7rNkSX&dJF-Am@W5)Pc|UY)AOE_7tl z;nH>3DaBr#*Xp#t=GsIq5YPU8`PiG#cyV*a#?CfKdbDx&m!GYxpknV^Lec(4m^tH6 zritMd_HGJvgR65`T*mihLRb+P_K(fP=!YMt75L&nNLG`S@ z%@Oic7(mb`byG2fn*bm{Wx^=&X6w1j(!}~*mV=N03SO1tCS${5SkZrqV-QPShlz#< zCde|;s%V~h5&?>}Xul6(6Z|2TU>ax;eaWu;g&Y(^e3>)BW#%aYK{0vA_9Ss={KGpC{l z+UXbJqzCnGfp7Q&@sCMhAzoA@f0)A9-F*2L>(FE5kstbE8nF*22JcZUungm6b$6W$Al3H*yEvH~B_5`*8Y1Yk0v^ONrO`sF()D$F7%}BTGxXwfO2e_zpiB%QW=J4!FFNB#OQ4Vot zmLw2jg(k|GLD`qqtz`GOlK2&IcKY_p;rqEohwtPrK$Tlt|LvF0R0qf67Eqf0df+gi zhZ5DOe1P4RUv^L3Z~LAgzh(&7_*iIqI-w}u%JjUm5R5A>Y2jZS(?l^kj-v<$#yHs7 zM-Afh0YjB-x6FZ{CoXD7lCmtSXFcLGydr&RUXEnTB@b%T41#kE<<+@fbOTEFEL~0~ zpUT#3Bi|_`*u!jy>I;<+M&I)lze^H&ZN=nf9%#00*e#@}%q!*uLn zQtI(lgKkil5ub)0HRK^XQLDJ}PZ_f!F+CQ1>at6u7iiueTl7aRW>D4jQb*UA;mI*h z?4tj&a93BL2p?OX_TYy<2~cCr49Jm!_V@ZsFtXgf^BcIt7jQ)cL?o)Nm9p@ZbO_l5 zf$Y|nE-m8ew+l@!!4_QXRDu9U$E)Zx6WfCvxo|E4kJ6S?N)}* ztM_7{T^c4CEr|zzSlL9f<@Q2Ii%eERM6vui@w7lzYu~CfL8gMsp>{32V~HBP&Xmea zEF)edK@q(o|8O<)L7W)+H%j_~EyKF)2n6HmwEdY3)5~A45Dl{DPoOwxI)%SNOR*){ zT_JrDCNDR7zyJXY3%YkvGY^>%tI4H*CwKr=(AfkTBQW%GXI4odCz*%V5Cs_^eNKSR z-I8nIg5*^c$D*D?cgPE$?-KFfh)(Y+;~gE5RI-L!cO`S+Jf||YT%A|FoRo3527<&U zbDoLZOv!E&7~mV=kgY(@Ojt|dGiH2Lhb%h{O*v9WoUVac;q?ut;*ZdRTS@jI%3>x? zF^1k%rpb>Y;woqLd$q#TX4dOdaq zLOp{DsvSousBUJCrFEJ)>2k&@R?V_ciXxKU*O7*Af9mYXNy+6mL)HNAAMW%LO zdQ!&z-!UFd$9Vyy3bckt$&-1{2BqM&#U({qWbdvK8-J`Hot`+u(zpcrpEvZ4zb1a) zCDz>U!kJ`TX0pWpNnH#Mz5F~en#)zpL6`&L)bt#)8@?O60F%!-+8;- z+Zce$SI0tTDmf^P1ACILy6k!j#_b1SR0u|Cz&;C!e&IVsrC z+^!|UbJO8&$8W)UObpA4(4&XMYC|SQhxS@|dWL7JNhXx>6yu-;<}h-}2uOdmY;cBK z)Q+s|@m>C1I6m0&-geEAtFm+WY<#NUc+&df938eODDNUQL3rgj2xz znbzn6+BRNpEkxXbq|Fw;)qicg@KZHx(bB&9{m(T$r8s4x3Y%-{UUqF^9_DjvWjtO5 zsF2OG7L$Y=Qb}M6P*qm>eB;iDB4s?O*uWpPQBlgM)RN-jq`;#pv*|86#^6=fRKvX) z)l~CSW>comf<=(D&e~+WvgIX_4yCO$xsJwqd&jmjt;T71AHz%Il=O!Hfg6Xv8s{=s zs{G?v95tftf=zC*HAz*khUAqtUQ=)b(kL&^rD65|6#$Np2lGprqAH}8+gVp0Dzbx~ z)0w);88o`o#B0iwiQ@Wnag!I>;hK?+u^rMu2>H@FUIdR~#te zG!!F=t z9l?F1*4uPe2T$6^?>Azd<;tz?#RqO$dd`#kBCsia>%-7PY%7+0jVHU(gCdRWKSzKMZqVJ?DNhsdFY^vxj{dbS zw9R!HU%;P3AAINfBT(#6;QtZ42TzborqrTWzbn#52luQ)q}Sx4n?Sq#1Kc-C2qzBg z3J9cQ|78w7-{8wOuo@9?SakG8;gww8Q%_V~er?r;EKBn~M96uJy1w1v&9fdU9r6k2 z$2v^h9Mxg-@%=;E-rfh2t{w5i+uD}SHnQ}K2I3q%%x7H}!hH^+nQR7ID{}}S{wGoM z;Xrr$MWjS=TE88_IE;6%v6eX_`h30v*BQ2EEC$oh>%IphQ?7i6HLE34X1UGI zTXkC6K?*-Atl~wI(XW018Rt#uRE=Zk7Nw63BTnylQCi9c6k)_uww=csm)hTeTU^H11NbD5^RA$-qlhZ9~)lO|gq5njn`k zN^7S?-OM{0TFMz}8nib0AIs62VnzUuQ9YDy%AU-Na`+w;f*~t7U{SIH*OgnKkd88@ zOd_vP_Z6COQ5ZVS{JIoDFf4QgI5DixT;v!dt$m8P3gW+aTK~Dr9IBab_DnO2lgDQC)DN;Hs zns93l_TZnKN6edDKN`7SwtM}XZo3=P1H728ZZ6^W9B`3+zrgL>G_Irq`}SV+znwhF zD^KUb`ZCcsTB&@x?CGM-9tgYe5W9*OR4?n?T1*>uBRl2`a!bHq@PaC)FCLieaJD5m zd4~NnDqInx{w2tZnZyMElMT6E4~N}7BcJi_mi2u+1liIJI`aKKoxPDrL~l_5UPaePs=PX-r&On(!N%7(OytV^;t){2_5z*#6GwRkPI8H< zOw{3lcq4`?IVYq{Sa$kiiVZStoMB))r z`D%)8fwfxV2P35x%<91ux~)g2t&8b2{b}fCYSVDkYAf;4M(UsqA6m`qf?6I@2A}B; zYDU&$noJR=dX1{1mcKqlk1ZUPasAS;1|l=M2QE6jU1eny)RQ#}2M^A&(Iz?DvO9M1 zTD`n>?2lxdZM!JoePgA1Fx75TO2~ZE8;a=K(st&Scr1pH?dCVGjeOTJqyL)RXTdh$ zqnGp|y1raeu${J3MHpA=B2OWw-E6`7Hh|Z1QxMFIzMN&tOnQazvNDC zk*uV6g2)qGj=Ucr%ZFJsW>SP|%zj6p5)r#3#j_-@h|54Ybt3}zPXXh!ql3qVQ$I(6 z5~n?Ife{2p&y?(d06KOH5cE;hr|<8bSo&Ye5Mdx$F(H_i8ap^?XQKI)?OlIp&#!3 zR2jMMEvZ2J@?C%a_^i3S$PWrur=t-z{C(mexjF^ zY~yCuj|Dr2!E?<22+Jht{j1| zfX5Um@;v;WPnUqvo%^GCJZ%Hbd+i;bxU>kARWi7FS3YZ_geSqgOCf+Ic4SHnuzhl) zIE%_tD_5mSi1^2I!-&a(#DHDsv5>g`dl}fpAa?0sL_>R0k+%J!4vAq;3(f{e z?~mig-eO%0%%iAO$Z18CYx$Fj?HL7v&(cW9v7~P38ji9qN)qIRddFOOA3~QEWF``V z@SHF}!?@9PsP(96_=lm1$PGg4|+xXUEANH%(5== zHLZ)bZmL=iGV6-cGjmaGqx^GPZ6YC=pr=_OVV zC1f6^7f3lL{Tg-0$cJ-52>w*~OAa4kZQg8g=^67gm(xRa;O1I)I^X z<1bwV<3)f{V8+r?N*S+*cCI-nRF~WJ_aCn$Dna$9L=d09^Vl!`F&gX6(_4%_tk^Cy z#Qdv19j(z(F)s<4+qvX&!uw))Bfh-;kXU!lu0Qv6(PUXJqOSH4vxo~Fq`YSh>hnYY zy2{iwIWqFT8;%zxSE*nSXN-ZpfYIUG zv$nNA#!FtCG*5mN9vl^pG}&oHug$jYiLZ}-iXTDYWMBZ zV$^tPZEeK1aZ?0P*4uQFER>p*$w6Sto-pV?C{G*WlRP~+qB66IADRNi-$WfUfjcrg4((JRQEh3 zj!8-7UOsWGme*kHoUmD2*4jDMSHg-r;V_E|_8U|3!7M2!EZ?&tGAq$ykL(A5Fs0ZW z>r3Qw$(h&6K@+wECpVf}Bwr)c>)gRy8+g><*|exFLLeD>Tu=a43XQI|Y~kQIeM>2U zQYnT=F@5Opd<(UVZ$t73Xbc&zOIjb%AZ8IXog(~4YZjGqT0%?H`% zifZB2GWk>D=Pm!i^55g?VrgZcj!)3JjV*D7&Z}RcEKo*H3*6Vc>=hK#zq)Y02a|n@ zO{EN&VuNRR-IwmTG?&i?o6ua@NxPpy`mx643%DL<0V>8a8y4yyEfvq3}C=WC{H z|F0EJ4fz$>qRB?r*32zSEL}DZOV&`upgprNNY~$m9j(zu#`^e98UlbwVUT#FL8Z)- zpHekR#eljn9^ZLX zRfAVrf=eN(A#{!j&Uzf35ds1w><)PW^+*$7NJ%NP^QS04VlgK1u7T7rJFLJnqYEQ% z2lSebFW_y4K2S~N?tl-rYIkRIe8jF#lyu%Qq60Z?qq0O(??UEr22G*_+H{s4H=ZcH z+HX6#MG}Ammz!rD*Izc^Xw>MbCR#5mY7lzKEQK8Rr*4MR;VgDGiSoPodycmygz3!5a%X;dZJZc{ur?=gq5l1~hl~~dyEjBfZAncD&$gT;x14&nhy_Hj$s|JIo}_GO ztw^cTTU_3_cBW5bUs|b7d3r7PTk>ejD#Wj>^A0hbSL))#tJkReeKw)xCws!s)fkPJ zrv{u&4kkGbcl9r>8;HCAcA)urjO+}-ry z^Wi(_z=1Jijt5MR3MNeh_54U0eQZB;viLS1% zCbj;}^2D^yDl;Mu4h`rWoD!dL_~98L=KzILeaUF~zIlp$iei{V-#C>s6qIKKUfB|u z+j&Bx0J7hG3cM*qy!?!$pf`f9z5f{2a$<+WwVrZjZYzVS!Ndt|^S^gYC+BJL%~_if z&2076(c7I>k3OPUKF+pM#Bx^qEfHQ)HAjgu=Ohn>zsUc;G3zcz3?V2V2(V>Gw4a? zh`Rfo%vD!=+wX&2yw9x#hO+S*RZlPBg8qpjpN3v{(MjzR#(=)cJjA);-_gTYyo&Xk z%EYt{ts9ch%QlO}Xa~`jmkg=0z5iHGw6JkxvIOyHr$2A`g_?_DFzFOT7Kw6^WT`<# zx3V-R&G|Vsb5=FcwAxQ7RwpiN!gSrMAg#8SD&>|x*m`!0SRE|drpSuciUPs`@+4s+ z370PS3Tl!DM}S`z(%u7zEhSFW#+ZKOg)9&I#ODwfC~x?VKv1 zY**W85W|2^g|lBZs?RmoylexCZOEGz<{2E~=uBB)=G4;eh)y7qpxUz)wUrge%%R@S z0aRmH=dN@g4j^dehk6B~Ud=0K1ybc!x7g%{G9ZND|kIKRZkJ}3mqu|}DOn^zfA8smQh!bWz1Kej<2Z%>iOdbnAYF0C8qNRz|Z-<$RKi?mFq)#cPiEO|^} zI&@LB4Y^kjBjzfK5FHJ4h;c8|fU>GVtQsx6#YQ?|5e7vT0qCQhyJ@1gZ7aRa}_DY zLNy;Gz{|2$HoJg1fQUDQ^oco->am2FZc?Qy597T{Lk zKBARb-RPH1ACNzOXgV69skw-wygFaz<#vWC!g*dRc;<0QFrPHeGPC-`H0H{e zCT4%z-9Oq#j?+J!!-sHNsq>JsZg2Wquy9|(+)D$vgy%bIn-ymGGz7K6ZrF|%Y{>JK z#wdyYoxeoqj56yaHY{bq-7JZCM!n0PNKD~1Vx+R3xd0wvz>L!x(JVFI!@rD!NP#@} zQ#OUn<{ot2KElUa`lK5w%$Gjo*}a*8Y_cyM{n)Qt9H{iU-B-Zum)RmBQZUl9fY1vn z`j-?!S4jeyM&D#?Jh5VQsw_lXDqDxNCZ~+JQ`GIP6w!uNzrD#cS;r^Z(kW`9?+2lt zzV8ZqhyE!j;HFypB9w1r|1N-va;fAF>ARf92kS@$dPCly1JRxYVnOdr1u96Gb|?2$#F$6 zsQRvgm^KZhD(I9|{QLdv~aR|on5XLQ87!*I|U6FUu z(q{cD+y)!LQigm;S$s)BH=3D3m#LGO@0vt+3h4%+Elw*ct7)_56r)2{;}BFAci9~* zm9TdPYUotuLiBJDJHTwCDlv`StmbsB)fMtN3_AU`O<}cG>1#AUGT$)2Ub}x8Fe03# zWIHmrS^cy*D%%p8?(7TJiRft9_<3~O*^fxZq1Pgm)vU^0I&!Qo3=8lHJ0Dln~doJ^A;x!~v?aSZgW)(h|vZANpAcuac z#f(n2)NMvvC;Omt%}vEvqi8#JoJ&wb=gcP20`CYT8fBalv?Coy_>>*oB4X-_Ez&Cv z=m@EeMP*btv!;J>7Fhk1#kbjp?Oi#on?zRssv%Gr*=3N|k|tY8E$Yx)N1swBPERp$ za53^NIR;PlfKmHbJ(#WJl4Q~Wb0SFU?!{x>!Eam_qovR6P>dGvUyR4>6X!6XXuLj? z*-Zj6m;{rQSp)=~DZ=kaa78rdp|L3mz|5jsJSXh*U|G{caK{eb9x?oeowux_N!_3t zcCd7PmhQd`iHU)t4OmdQ8`ePBp>9K2z%j!Ik?+`l5u*I%2ky=yJy}tI5GasIS{d`6 z^)7Ws$4N!>09sZPFNJx_gxvNtKpU;OhtUxWHQKo9^Wb0iaa~%^=NF`*8>vMC>IU*5{9e$$c)mv@@%SSQvx4zFQ!f)U#Zi3@^ z_6Vwe@2RjzV7;4iD{5@2m+TFBidE}>_!JS;ls?yp6*N51rnL5*T1LB^Ve!GtJw`4_bs&Qg z7_77vmm%2tBTOjjOJXTN?ZtRL7X`a0DK*FSFyImpX(?U$Wh})<6;(h=j1z5nNR~xl z%o=S$jiSP+X(C z)5#3vKL~t3zdl7e{*`0Vo(&@8_nF{C%q~vz8|M%h1BZ#e1AuR3ED7@x!*A+5EK3x0 zS+bu45KuJ2Y~0v+5Cbqm?$Vl9UJ(sW1Q3tO^R9*Ad*)F+71iXuI52mZU?{KbM384Z z@Tm94oxUnPSNZoPud1rUhb&PLR02N*(DM9|&TzWJsDNRm;XlwA+k!^yu|J=x;&8^q@QA|GLKF`Q3M%`5w!6V za!+oeIT~fwa~x!CaR+EoM2*8m`W2b|YbRrd3-4U<@ets{vRYd~V>=EK77*SH*`ZO2 zIgwx>lJEM_TTs!O8PnVe9IqnV=0wk6?NgtAP+nG;j{&UQuNaNz68c70r;i>SL2852 z!N@81rvFOR>Mwn-{N&Wx8b=tz6!FRkM5=|1KuPQ-%-)+1* zdD$rl>l0D|xOkf9!yBHs1MV2xgwJp6i&63)ik;!6ag2_vZx)sWNkFj~E~>)=^{?gr;?Tl(3$=(0u^(38~x!c+@{*__PV@M>?Lyvg%h|oN^6rd#<2r>B{kOwG^ zZ~)i@Tk-2du zq8C8v<)vWZEF%|uy)e);rFS^^Py2&x$#$SDBcwcoZ;w_Jl0g@^Uu%&ECl@~dfLZ6{%S8Ch zB+%%?|qeFQ`Rn;ZiQNVL9E@5XSQfg$}s1ZU#H0H*nf96&}i4=zOg|-zQ3Q; L4XWP$LV)~#hz?qT delta 15644 zcmb80Q;;UW(x%(CZQIkfZQHh{^|fu=wtL#PZQC~HpL2F2b}x6YDl00YBI=?tGT-OT zu@T_W5#ah~5P(kIQ2GR~Z0#y&g9=s|GB<{8BRy%AWl>-jAGm@!(^qBB@GrVQ<5*s2 zpK%l~YV)q_h~DCJa&n&vn@nvN4XQ3}GzTacqi1jd$fj$c!cU^CR1~Ov$T95)0IP2G zuQKbu>WH0B*fvq?S0)D=*;V3vw zWrmT4ePw;9>_Ftwf1)`2%>2OgA#rpA*p!$eCz69UPc4}qImq_(t-!URrjM|%4Jl)mSNtIqc zI%jKX0BFw(QltXrNTmBl^oqIb{d*;ca217V<8tcLBuB?9(t&#{jlLO?t02j0{!l2) zwWI>WFWnUE2)(0oj{IARaONY>?ABJ`*ml8cV*qOvu*hAQaz!XmpvVO&B5!+!u>%a# zeS_wHVVoA8<5fi4tjL)jvB0(1Rc;v&3ZdYgQgWs+$MzWM{%w!}`4;YMo5Wv1#Qq%I z6NT3k9!HC~IQKR5%n-o*X78Y!L9ULhOHVrTBpdl z1Yi_f-Ye>aAitOfHSM3o$)M1bh5$db5J7Ux!slp(Rx->BI4^vUQwIAjbJ zw{nVgLyCTs?FM;~ig8!_n@3{^q75(vCKmMF@9ARWZQ;R$r)zi%Grsee)7t|_iMw`b zJS?aU_(AlxW9qdZQx?HW_Ra9`v!l}S6hM&(9jN~-v@k%$YsQW5`3gn%xdEzQ6-&Tw!GakgI|> zqL&f4s65@$+|`T7B*E&Qw$%2^6#M8Bk|}ae7hzNr;??Lwrn&-r9{cb@(sKl& z6WL%*Bno$2Gha?CX_gJNvNUfqaE~mq zQn8#77Rf+pT5zIZ#i0paL$lnS=t~XTEACVoQU0%Rr#^B_pG<_2qF^IJu>(?=iFx`5 z;|988IKLjxPDBQNyu6?|fCD4H4?oB$A^%@LXE1&M_xmqSPgp+zfH&BVFF)w*uD&iF zzzdPThfe_T_;bz0`2)7&|G3x|HhLS2*xx^XI=CwuC6U)(Hj7!PU%MVPTL|9tvOj(+ z;Si_FiaA<#>cGe_ehXKkLrj^qjzk4aDp3Oe+E=6$++{26lkCF<;NrDt^vtJ-GS}Df zvNNgVgwll2PvP3b`@OGrzfb(~ylclU*A!$g{G%soaXr89J+?#a3N(R6ZrQv zK1bX#^bkSRFF9A<{o4nndI9D1?0Z{HuJ~Hw0TS*2A#;o{0#;K%x<$J*PM46%cF#Ub ze$_)~*>2gdwUVtDK*i=v2&W&Xu;|^gS$*Cu0mi(gH@f&-)xGqjH3^O|8#2=Z*;P8m1BwyoV0Tj2Fxt%8YalRmmLcgoc$p|Vhh{ZCr2cId=ian0uuUT za{@`{z22Sk-$gWe>~Z1+aTKfb2s_VIBENjNEs`+XG}`ueO&Wb!Uw+oJ^c)l{o+5Xn zzcy(-J0~F!1+*{L5q|R=ec#n8+S=VDc|g|zb0ks1D1o$QC3(Sc0xvc-!Qg-ZcQ(zy z`2hdh$skyP@@{i0)F*qZF>rdH&{u9U+k4m2OEc`P$<*7(-IJ*2pcp$|K#`wO9WTTZ z3z&xLPR4wi40{Z464HKpD>dCAPk{km#SW+W$D5JiQc+9_{c7GC-S?!ktW(uGx@$qC z7ds@GsD)7qRNc2wj3?5}%;zMewmd3MK^0mXeL#Y3>(y8)Vs__P0>`*eIW0iKu<^_t z)BFD$pUUoKLleZY_JgW~yx=&z!>-j;mrPQLbcArbNp`oYgu1H4T*T8f@nA}T0XH@U z;evnz(&{p~gj#iZTmq{j@=C5U*s4XhY2g;s_b2B|GQUXiN|mw7|+<_D_UxLuC2R)HYo4Wcbby zeJ!Oc5nz-k1m%*R?OLd{LZNK)X&>N=){JNX82Babu<8S-9g?iYB+I;#GXbROr=&HO3KFE>gMa*!F!io4$@spk2D;YM*5Uk&O;>6065(MCD0= z8N+zv+xbYMGrZ;%ozj!xjMD0Mizfj32}hg(5$>{!Am){W>H;*dN4p4}OUq74#-CSc z$jamHA@vT*!txlF0-VYC3g+abFF=DR>XU z(q1yKnAh`{x;ZE?R5T~YtG3V&PEsB#sl!0Xf)DIpkpJY6jrvmLMooGG8c| zM~@sjWs#MwjFz5hXzF}Cc%IDB;PI{tCKF%H5dMV;W_fMhwtk~%CGUm12TcHeh99%! z$aBo~+YrWlxOnLZOiI8_2Q1eB{rWK|d8MyA-O@#AGk?al$W=$_lm)SweFS|tVV5|{ zL%96l)3-)Ah<-~6DHOmC9=XY$;4KiE3;`MMy||bzw>dbZ`^+8x_MFAf$2p1`=u;EL z3$D=P*W@HP^*q1lpWeDM1Kz+H9QtYTTUz{p5oyGhMh%++5;9uxH!B+|`vis2r{VP3|30OUnlHBKuMTe$^V`>`ZK;Wgq{* zbaS!_@!`rQVSGfoBs+1DW0UmMJ7O!$u8g}sx@@c4eKJ{};0oh4hc)x99TjIbSoZPG zFu>afamu4)6iv_^f&+66=H)&l4f!-g{#Lp&ZPc1{D{5$E z+!H&V2?`)sK2fHuZ}p5Md&f=^$BNZ(SR30}3-95pg}KSKeQ!YpmmlV}R~wRQ8@HEc zS{I^!8`+%-X)~xFHN;V_NIv5g^o}N48Guc)g+IvCeP(nfCO_%q{1z~|ui=)kTw)1@ zM5mK+4OUmKO=&fp{p(mx5?7PkAW&+8go}#jX$bJQ3Flb|;t#r+H>#5{MRjYlrpCyE zjC#!^v0?Xu!Kzp^2H_LmL0EFM6Bk<2vb92$MztsDYm=^d5C7h{YkhA^w+Sy#!LN@5@e#;J52zfW*b2{{23EXmab}^gWC;Fl;-K&D>8D<} zRo@=*1o=ezFe2+#*PJSw3vY0&NyNn$7;P#33&`{C+5t(sVDI5-H6-0PFm;U`XM?l( zk}jVXHOwPT2fdK(P8MdZ&7BV{lYv)vJ5NA{?VrpPJ9i^f=mJ<%uZ<>IJ@GkuNYNkp z#>57s-yoA8{=^4GmWPQxCkrj9?L5y^0E({83iL|~@W7?C7)l5dX5%J^h* z*}rnBXlo)}hih!!F+Zpc&Gd!pX+G)JhzgLgRmF8pDOEQBO4>mgbVQ!b~SiliYv=(T^uc(U-_(4*kNHycPj} zOd@}u456Gx7FKNj@YA&>lnmyA6t(TfDh5`Zc)cq7{;so<`(=g{P?S+=T){mcDx*}USW9;(kKW_C_Y6s1O&N-)*W zL9ChD2-_Ckomgf84^qpibS2t=It-9U<59YT8=LX(Vl&^ciL`f#1!+Lz2_RQnT@aJo zh#3VA@o{MD?99A)K^d3SERo&n(6Zyw91 zH$@po*=ObcGHA1*zhVW!#FA-@#Xg|xG^(sy29}puz174E1y)yVy`JXDYEt*;p3$q* zd4A^hAbmHzT(0WC;)GM!dHcy?iav9L6v)6DDOltl{{!HXbb=f+{1avE`zEBgm7bspW!T~gS!x=Lz|63I`|kEpHV&T zah>=N(iW$Gq(cCABTr-<41eyi>}+k_7Y79HOW`ES@`{OktLsV2?m(j}HkU{NvvQ1M zDpW{CUyVw1^pTIJgjaTwo{49i7kxcuQ6H!j+sjtwFiJ{s^=#ogW`&$(ysqrAngcW}TdD`1LsuicjzmL3jy=2#Ek z-N%xy>DXtK7pqE2eSOZoqwfqHR)+DE*B+2i5L4U;Wc=%_>#%ki1CG&seTCm$htwo{ zkcVxwdEaNb4L!xAFac1vf*qje2*q_s#XM`5hIK*R2rt}go9Hk0tKbzXhF7Xa*|mQ( zGz>7_QuD4Eg525FdH(-Xmz#P83mOH$*X&%B zytB3u(5^Dkkwsj4$YgL{#;yJLN{m=c*L8EPbjf?7;6g6Xgw*Dph^<}T{@Zwt&VA#+ zl#tm3B#s+3QF2-1KT$dl!0yn>I@X1RSZdyh)m@W_cSq@t=M>ruPb@aC=x-l(Y^Rpc zs-CovMg+AK;dLQ3E%9SRJEzGC`rx z+B!jje;k{Jb3bI3#bs7;kzL0>MONou%G$ld?V@qG7u)}UL2kO`E~dRYr*&8UIiNEP z-d;*=uBa`*&6dA#RW3{7&UqrfC}zAk2xH~d8YpP*{`Clg-<DeCKv))wX zQk3RWXaHEbOogwVd8RqOxc*DK+MYta`NIi69C)r)mjbVau zQQRll_^D!;sAWm$D5i2S4p;!H6+ZkEAmsn2Bo~nQ5oN(}B<)0HiF;8k&OLgW-xLDE z3|;O4{>7Q-RDd#8r*9m{6$*^iFL*|%zRJJ+CLK8e#)2ficBxbG2hp*tS~uB!6oy1O zbxms($S3AQ++L*aiq+Q6-p;<}=%iFd=tDzj#atf0&jV-|*az~CkfaZwg;|`tKkhxW zs|ygp>r?VaKy$wgexcp4a`Eeb6k5&;r2JjK8@*1;?Eqzxw}`ZH@P6UM0A*?*VvMq+ zkF#V&76)~<2K3Qe52gUhAHsXj+u84PA#oQ9gaeF0GC)|dpVO)MwPIN6-@JkrT0;x# ztA;tCV|T5R5?x4py0n~Tl5Wt;hdQlidJ2fauvK~8%6=-Ev&GEKh?$F2qcA1MW086> z!Yi!|CJc0w|5qE0!26luuLI?Q?^ahslY)&n#dBSLh|DkOl6z60fhEf-x=I;hkzfT- zC(3ukj&c)pQY}gv)v20cQ)6n`g(6=YGA@VY4m8sqJ2nW(ebdUqH+b|Me9^LmXzyy()gjfr6%XeN*VoQ~31O4j>S$3B@4^d0NbgPup(PL^6bkxVzUvg$BkSY#G(h0N+789*D&j@efB6v(@9}~1L z^5@QY&d6bG817aCkGHn+C$TAnBE~Qp7UriQU==2pzQ22T@xB;xh+~@>gZ;K1G^^G$ zbe@OFCdCJ77P>l;)h}rfTXBr%7*Qjd%9ygjA#O5>x{iKwGj-$d>|DL2xpQCgB9`B* zY(=Y*B}^A!=hF}?)LlJN8V+~eOq!oapIp|;K#W9HZ%?aCit70I=dO73S`1q~+(rJm zVFP6Q9w7T%;H;h@OL76!WB^n!D(p;4&cTCheH7qXHCXPilr!0s*~owd3b=O>d{o)n zSed|xI6f+RWvM-19AM#gX|`BO(GO%<(xFneyO{b<;BF!0hki>0Dxowm2vxSgced5; z+=Nlvk8V2IC+YPY~A>;ZoQ5Ki@1OauWs03|JaJwvTmCwdos_7$(3_16~-+ivs`E{L1@ z{+s@{@J7hIF5D@;A;80b8~EiDZVeyI9cfL-5Hf&}1Z>9!N{bO_g~fhO{9Emh>{GDM z=Zo)5+{TW_Kry*nj^QsRIPk!2`NvNi!acga{s|_B^^A3+4ajh-)r<{qe=~e;*R#0- zkuU^85^f(YR9IbNzA=*(94vFSOQr(%kZBl>fZ|n)lX2=& zakK2e#L)Yn9}WsChtwKL_!!$6N2zQ&*7X6aV0db@j1gpC3>1FeqU!UZ-=W^_oFT#a zL%JHVr&wmXUe5 zcL#G_9bDI`)rBBNq(>J~&Q#w7PN>kJzp+ZTE8jsQJ7{7SQ14LKHLVhM)}nT9Q(pkb zjg=d7U#8P4lkKl1b6rhmpn?7dM*Y}^2oDzgoi`}E&)ec34d26K5sw`c;)WJ2)Uu{P zR)VQvn$CP*sHox^@)^$$6f$V)Dio>_#jL{|KGl8a-dMhDD%Zlfh=EZmm^Nz?U+G*w z$sj?vhNP>-lxub}#82F?H3O<~*FgZhx$U+xEv|cjc^Lw9$Fpc^_-rm?y^LyOfR9et z8e~F#=(q|H6c)_LtXc*#1!{{|18Lhjv^hR=Nx4?Pz9~yz ztE^b_ayo>gm3v`@US^1@g}zR+`mGwlH@n`5H~VVY@#%N4z5Ko|(WiYBS5xuEdUqN5sJ`jfpYAxX6v90T*%{=tXItJ6hC8?%#oG(gE;}t?kC}`GB?MKPb_iBl z+V=M}o8qI=Fbr9I-R+cxtY_=2ujag(v}}qGzp&{i&X*cj^cGoCTsg&41RpMnD3?pOMK_k3*6#bsg+c2@i|0)&)Vz?wo6utYC? znq`Y_EM~4`2z5YhZskhnX1gF!1hvTU3zc3fi3M! z>jRY??RW0*S8!Kl3~|-UhD@#CR;F@KX}Ozs)46`@2y&RpkzP(%_?zR>@|)Va>aHfo zG@(EtTfJD_@5IW0>JNSV1{

gb{A6OT+4*hW`ct!>jH7wLD=ZbkpU6TR(B$F1O(V z9k~KRSt9a74Td-&GuFw2*)kYoZDQGVBlZqm5Os<{$9_D#WLL!nAt^|5{N%1Q#YPr+ z2fSpXL_bT;)w3HgfzDGR%TDSu36d95u=hl9&UI1TyCea0d6$oWa6R{UgEuK}ZsURC z^63;{tBjZ1nV}3Fy)sJS$B4SD_VKKG705zV!(a+3iTc)YTWGhqqKjnD0VwpMUB8Xik%z14A@Q);M^Dd&!eWXNWC3-tZ0^Ss`GFR6zmk-xyS zhTeIRz~x{?cKyuG(N(C`+f;FDp=<2Euqg#_pzOYFdGf1AR%$h18%CB979A?W)|dCM zqEJ8iG4pm|k(j1oP{atkF!_^o)~OrD_!19W{s%FHAit zWbGVp2E5?t!!(!yDM)Nw;%VBEQsK*hSAAh?*)~62$r)nOAT%RjJA-RcjPNFcv8Nh( zv(nue$>$$G|qDaQaPr=4j4G(F$2pv2L^k)xs| z@Q%!4X=K!q8M2AR+V{ZZvt00bHZOIs%>EGr>x$(Cv&`*emDe#l!NNb@n;)ChF&Y_G zkLUDJ#2^^zs(dECG@vRzP_D}eiBEpNG*dv)DWLzg=4ab?YRl_;)K)4Q^qIP6Dx zA{^$G3PWTGRmxH|+1qU7O1OT6^l4VQz)tlc!GRpZ2l7Kb>UA<62Y7tJcoexP>i;=$ zbfg%zX9vC|{>+JB{L*Hw5t%Rrjc~oV<|)P>Kj-p?>lQf^oF~uq=@FmT8M%jxY%%GI zmwZM;#=2{NKLepm0)&n*OKJDy<3U)X1sn z=@)OK15Bvwu%r0q4byF|ic#y-2D^ju6t&RG^MyClzr`3)WIK%D|+q1c-fxQ=i z$Lc=y=a+dk`*mvPn-S}MnET&rH@OW&GmW%RcLT3Fcizky_68#KHr}nV1?O4AH#OgI zhlwK!TrHn2Bf!g=BZJ{!YX4ILBo?t>P&{|g z-Y-?c!ZDV;ySH$?#Hd+`yx0Es&I2Ed!{nYTVu#`Bs^K{&=JXFLkXCW@g^+DiCkxA zV!GC)2#Knk2yu5K;N8Na+uYjGB9VIR?}(?HH$cKPBWeKZ-cbIU?pS=m(Lp+}bYOct5|LK) z!DPKBMXw`(0ORJXH3ZSo&q%$4>14Mv8C|TWRV}Ge_*$47_7LIzerkdq&#%{y_C4uZ z^*!l4)r>LEgkS{YOJ41KzKITh`=HO03#ORz3cxAY=7~nMH@_~+gmr#t!yR$VI|ue! zVQwD!iW!X*ZVZxd=|7TGQGGuu;Yj)?>m<}jw5}n}hbct^wj^yMjhdt{NXY-o=w>kh z7Z2<+?TCxBU~YC*gAEQi1`Idlwu>_n!8Z4;(l9a~@lKOvqd3))#)`+qcaf&>#hmtu z2p}Op_*An`>LCT>iI9s#qmu+W1jiIBd_49`*aU~zkx#`Wg|!aD_U=${44%C}vPK^V`d#RN2#Fq;38vg**njPQ3=@O~xAx+K zNvXU$9Hm4psvr^rL&J}92^F7XAt^bhK;H0Ngv7F-i{KnOlqNuM$}{7eL7gj`P!1*4{-Wsjr~M7C&>%2glc>FJ zDAA!ZO`HYzonY~N3{#zlnp^_LdPoTj>p<4Fz_pOoG;yQ_Gj`xrbJjQa#e7TCHs|icn40Y=|gRpFg)Ki4kJ7c~Y zH(hHTlt$598ozw(oxRf$+NQ^%eqLT*gY0IZ`XOk#D=(KTHZ9EuxFCgx;+g2E4&mr$ z2ai*Edr{$~g=k}(Fzd-Wm37+?3*QEN$Y{r+YOC^A5lt;P7n^w?1~VaY9!VH zKrEg1aP~ARf0|Q8LlU&C<+uuEse>i8U?K`^l#<_!siQ!#BwSAz0DQr;;)v?`7%z6e z6p9&0Y2izXNTuDNG-FU!fO)GcMAxgN&dA?)y@#r9C$*#iNyRVD@1VIY?NQmT>v=FH za03U0P8+>e$};JB?yNr{)UT4`XW|z#TPDjwc-HpD}=&sv{0Rj>Dr);y1^pxyBw zg02~9a_ctoOox-f{bLuK2Atm?HFmX`)oJ1>X6Lr-?elbh%&q54VG6+dC*fevaKv2f zFb9XaI1Xw7K0m>t;87^Oswromg@O(U`!*AH`s~NS$les~7YjMNWb{ks7<1tK!umM~ z;@XN2ZQ5iyjglHVB!vmYZsQs{?6NYf(y_Jv3$zfX6PT>fz;J zHJOHvrIeE_@@Q$qv1~0CB0eUq&kkyvI5dd#e6#rO%3f>JIxj80Mg<9zfGc)6*!XYR zWaVF}W%0m_{+iY2@jLlwcX<NM-%nd33Q2ulcP`mEWM;5`a zbGH=0xKm7m)>@ZJB*_qIbZrZ!W$Fh<-s1f6pc~%8@nE+K#h+!gv)g}1ND8vYIh;nd z5yxvrwG|;#Od1^lq$d+Cr(whQT1>;dW)mmKVTtfPX8NKI#rlrIB?I}0z*X30fvYnu zf4i;M8x%(QG3#efV)T4$OxBcXH9Cu{S~da4nhaK@HCnXi_e!6oxAwopeyJKfL-pqu zx!bWB*6HKUCpCSfPU*OtVY2gA&837;9%|=XW+j(MIgh7{UsN*rlj6A4SWiyMSDDSn zp1Vc9=J*bmI_|bByU+Kuy}G}+54kWSO&AjW9T^iz83*b7_xE1cUfMW3T92poe~1?Fr%*air=#S;9e$pzo} z3;YVgX?TB_RkLJd=eQm?V`w@A0;2yCd>qhXjzJM+WybkEVg1?qilwh4ciG+OEN0`g z(;4eYoK~~^ex^M*N{?#IS&eMqtf>XemPIvQD^_7>ZE{S7n2XpwXWlv@@2=IJUr~Cn zSCNL0@e~iU(A{EJF{ifXtI?a_GN23p?H_2$DYbc&UdtW`H$e=GzbjG9T%SQPtjTG# zbKihyrLXY89I*!9b1u%Lq?pF}GvH>VzTV&Y9OB`2VJFg`N>rzQevJtDA`1b3>wnnW zPHGo721J$SLoXEE#EjmvD>rScku$b5Z3(@tI?NVh?uJ@lvnI&*kDAJ21`VN5$M8qH zeY+|ySDo}hh@?Vu2o;OR$@VJ$E=jaiURYEzE)T|GU>sf_Iz1AKDe_s!c;C6VIyzh}tyFLy_PL4l3s zx&FEQeBl_fOJv2XdQ#Bq4VCv2UaVzG&&(B}Z3 zQcS$UIRJ_UY0g&BQ&X8Vf_XjyQi!ErI9Gq!1tOao=@brou&kLENl;weVU+Ak2I7WL z1Q3TBXc|9gI0~P3|KHf-;Oh!6QZtxjD#U;_;an!Ix@}C}fL$ygUgr}^NT9Fya{9^# zcap|TO2qw&^}1CFSVr3IDDz~^cHK-o{r6vPcnb-NJ_Q-FaRvC0Nhh&Ubj%C&ah$5T zl4y;p<(yb1SQ7@vfT6800EaX`gm|F0#q93>t$&;PU2)&&_BR1Om1Vy zmCHBa6Rou8fM@t4|7A4Vp|%R|)460X@sblepXy;Vp*#YQBAwW@OxE_SiI!dF-n4a$ zGCG2f+h%lFqNYEirKB`q%x?hKrjDj(!nb}LHC>tyu$T}d-oA};3qJ!bWH&~ zKyN1#M%v-e@5!M&iyRxtR}#t-XXJNX4pR%yHwdf|+S(Cbe~F)`z5d<$dE5m2_{Q*m zAtLVHb{oy#{&m{}__lvL-re|of_lsB8f=LZD%1i#3Hb@&AX(gFG zs~C@Dd!=WqN=ExA37tg!mDujl%2Gr5qnQ{0n5CO7X-t z-XoUS%!1x0Rcx}F|CpXI=BA{(Sh8(UXBalvnPfE21XN%EjBEgi-Gww=YE$Hftfz9c zMP}c&)Y@uSoMz~sSM$U6z75N0>S8X!hv`#yu-Xv2s;z4f59tzr{tY=4H({T^T&SAE z+5!xfMUJdzk`3`N zdua&Hdl>Klh_8n&G?~0XVvGw*6@tJ(k8`?doFYa*e6CO;XGZ)ACo%%+$z|>W)K4yT z$~Ol04ET;RRE`$8NOrzZ(k^Q5ywy4F7}ssDK;@rgt+g=pe7MnWt>!aGd|;w(?d;nX z7qbHvn=#0FrUT!mE{rC5vp7~k1IDs%JNEELA0PApyCnoCGd4DY#?oPKN(eDFRC5H~4PLsp6H1{doJ@~ezec!ii?M$&Nw6qZj`wTv8j~l+sFlp5>=*i0S;!sQbJ|o)D!VB)o!Q*! z&kY

#&|LTZ!%Hn{^n~m;ZXkBAd<1oAQqz&&jS9#kK;?6@W0iKP!k!;-jq#?{k+K zjw!GA{_jNYL@_QN1pbAu{}n04M*^0+45>t|UkX04Kt2Y28k9Y~g>zE9ZF1xR+U{-w zZ}X!6_DH&f^E-LzEGgOh3r|PaTeX#Q_Y5fDqFnbnKyYyD$(Mv^rRWOgt&-6LWltVt zOUj=I+L01?M)OD>#E+Z9dmal`E&%-ZVcAQEIId)>jg{19sIAH14sy$!_pW;E!dmq; zK6nQ&mZuX@(ZAI_||lnDwc z!&ISo`HShql~+E`H(>_Qm~>THZu=4C4vVHEi9V(wy{RD|%+9Dl)r})`OQSG^`~XrO zuOFM+x>LE1RKu&f{H2LL?F*ZQ(>@Mpsa0e~b~j=>f~lvjR1aS+;d80b;tE*tIe&3W zA~)4*YBs*MInj7JeB|HO!J4Dx+Sk5V@LU;FXb(tt13ykEGJ{2NoK0%F_V7Ws8d; znMo5n6IVv%Q@I3>VIYD@pOb?eHf4%iLh4aEhRBrPQQht{#sXRUD%w39#e@vG?DI+Y z(5O`(-zQsjm=JQx-Xg<1TB$aZ>Z!7qnJulH!)^_ir9-vio%jh3e$lOSt7NlmW7HVL z@pd52%*=r^RuD5tY3mj!j|2c0Xgz#V-R!w#&8uUd+V?b$l@3&&npuMs!s-tluF}X} zoy^8W@oIVrm(E7Y)Ea&!+R>epfj8+%NV;33$}g=DuA&o)QD@{Sfbi9W^OCdokUn-( zm*I{iInXB1=)@cI2(Uzg5xdbvGz^3gjj&-b5QRR{=NOp;5fZ<9cgc z--&g_0MQxM|AU%8udYJZq7c42cXpBFxeASq4X+ARRJIpUht;KIPh7}6!2y)%Ja`=- zoBIKAWt^C-Vl)iu%O;|V{ru}WVNb(Wp7#h;UWYh=X2*cQ{yazxqoI%586Q5{tnT&b zTmN-a&dB2l?>m4*-3?IJIK8lHw?>H`N<8CBwe5f1Iya2#E-nX|(8!;O-(DT;HEIcn?h5EN-JH( zcrN}SUN(0a72>*K2Iwg|p`x>$#0Gya*BMXIq=I;a1(i+yT@5G&8imz)^Dv+6wN)PP zZfd}ygpP4d)x0PoaKJsU&a`_RC}nGzxvLy^sF-J`{HwZqVzm3ZwP_a_L&S(M-TC07 zw2I;`9eQg7I7IpkoG1Jzj^>QR8}^@zN%%JYmivPqm*y++K$vRQC>q@ljRP{K$vbEY z!UXv%z9iMaEfK)^Pcu+Mz%Jf9`KQN_jhX;GJ2-}%fW!sU2-FEAm&GbRUq|q}))mgB zj|@R313ADjW9y^L55nD3Klj5UB--Qwp|E%Q_qRLWEcX4fx`bePfYrFR3WTaDkz5Kc zSX)HK2vkTLgab>~Cng*!Ai<48Of$Ggj;iHPtsVa$(6aBvNIT?IQc1#M+b92_*DReO0JWIzT1iS0q4%mDq@uEX7p z)e%B~c+yeTYcgc&18YKbhVh$Oq9^l(>+S9HJ;eE2iC%Rs1X~a=%7m0tlHonX%{BBh zLjE2Icvr<2GAz}>QR1hcBcezZ_Y8!FQw}tDWZ;JmME1W*tY3RXGCAagI-xE&7e(x! zM)OwDlJa3j+M@*{y0hYinRCG)Ihb^NFZEg%I+VGuC=VYsL4%g}@$@0%|HhkR_C(B( z_MSPi{zIn497+iWbH@r+lzwR*2&Bj_mhd4D2rmk+K&0vb3|Qj*KGPs%j*VdBMPP(3 z7!nat3xEec*cnkN>#M z{!224NsF-m;~^lDw8KM!qS_Cjv?e99)ML2j+*?C+$P1f8-J&}elU-4njs=)G-u;(j z4y?_d-dclI1t&oe(j1O|$y6EACZ;??k_a?LSklcOm52<2vk#v3MxD`Nn&e5}QNjEW zuSJct-V*clvrZjvf4s*3Q;Ly7*C!XDrX6_~5&5O5xz=`Z5aiievA(jvvi@=jq`&Q2B2-qdQmXx0bHa)`^hKyvmKYC=3++m5eNPNX%T#Atr&38o{C(zq7eVCo& zqyR`wvXDgMghZA;sz*}Fj*Ee%DDR~PeLZvNFkD!)BqQ?QHx z3H;oVNb5Qx?YuiYBxBM1_Atb9Bc^s&egGKf`d`xZ!^D2eHDr5h$=yb*XqMFcRO575 z@>EmJ^{`a`S1Ue))!GQPbZ{gLoNYOkiA7rCa!%RL6Kq{G!#Yr9@8gY`#OEurw z(D<04HTCB=ze;Z6#vwM>1@YV}(D9$2U?D*V*DmHO8lFrfUdD7_WYc`+%3NU{J7?5=#0!&D`u) zUISuM+qdHoRQpMfsBy!Yj6J(+j(arM9q)v8E6!1>Z$-QaEgEtc)@_Y_A|NZAj?<(& zLBmYd=mEqR{Fk2INwsR0AdFc}Sn5Jxz$;1G+GfC2O?}#UV8B&L+xiS(^7KA>O-Kfv zpniXf41Re9PP(RCCn08~{PX*pm3Ab?jBS+3xouYI=y0R33g{hDsIxhGraTf7f>=@v zCjP2ChocTEeW|T@ijtu%`9<-IVLZg?d7vzO+|*sgGe>vZkFLk%(gMRy7Vfw@Z-%gu e@TR+|{aw4~NQHoKJ909VC9T|ToYR3Ql9RR!&;7yd{INa^D76~340Oy8-a}K^*bd89& zwyhJZ-yL+vR>#6*POX!#7RTI0)`@jV`G$e(t21zVbzya^d*m@piTtA9?+yM-l%<5!EomH6~6c9Bgn>7Lr0dZ&)#A;zqZ>Sfn&)B(uzsP__ly(PCC z&jIUR5RNzThg|=D=6ThFL1SXk?6;o_xgk3$okX~pG!2Ld*#tJ@;DYP z=FnxRWAW`8iRN_nUu1LZ(}Te8uMPH)W1Vnsg9I{b58055C-Q%*2hFpNg}$gMJEt}i z$Rx+`4)f(^20XN)903=TJF8=DJl8t0mYlEI$*Wfj*`#aJ)vLKrZwG6S3cOyqf_7ic zTsnKTf{Y{Y6qNh`BZyG$s8|srdMW$9P7 z&Y|K&yV^7y;=O+tO3t6-^xI-8{%T!$=!~vbn2YK4sB4{&jq7$S?ZX@7tuTQcrJkQ7 z=w5ARE@sR08_eL$MZbS0$a@Q2cLwddfbRWwe^Bo({qmuRt=6$eU(?8bwJ=93`KAPK z2CVrIK7gJFx8k>MT+IR3PR-Y#-HcKfK@w}3o+w?@>~eoLhHrN98v_0C%%tS;C`U?8 zk^DFQ8`T$)5e$ud?0|J2q%|^l0cW@yeOD+nqo#R!KLNA4$V`kw;JvfV3@3fvfBITp zYpNusG&_d$Yr_x%lay>$GvsB!{3ag*%m4&oli09_qv7H4(Qq;xPd-J1^ILw`%sK8I z>Y4LwiR^#7E05j_Nn49G6A+AyIm6v+w7>#CLnt7c02_7_Glr!cUBQnB>|(3AA#UIR|_t&Q4E*7`7Om7|MS6TR*jXW0Lx;5shMy z=~Y7nr6iRww?1_Oh5WHVBPY$Li}OMip8P=4y>@>Jea-Iwe(3nKSf=fR#!!$ecq5-y z01%VIu{BcGM&%_MW|m0sW2rR}&|?S$W?Fsiki(-HEc4rz*&6f*L+iw{A^9H;pe^wQ z9)b>VseNY&1lN!YuD?nUNB5_gY3g_Q+-$sYe{4`AbhX}a6iA*LS|kR*q(nY`nr*Pl zG4X#F_kzr+)F5KLh2Xn}#GxyqEWfCAOx@(@ix>hq>Oupxnwm(CGtd%y)eyRQ+q-a` zBH4(}%gIa8964u7BnQpq;u8)o97H&t*(u@g04A0 zoix_4Kn}^3Fi0ArxTvR66epX$-g5Qv3ORpRA~VATdfPX|rjDSm6Ke(;8joVz_dMBN zmyVvOxE~xD#Jz_gN3WrMw^`Q{_b~t_aqk=9N@iONrcS1@(l9*|6HBk`~ncU#~vjsAuaErhvlY+Ar3eN zjIIz^L%!?)NF3SMEfJlGXE7u{FA)JYbY0P7Vwjjo3^0*63Wkv1-jaqCR0Z?QZwv0B zyFpfr-Fz7J#oBKh*VPDD`7pM53I%_w>wf3VJ)G#7`y4LBUJSiEtHs*U=lWBOkDr1T z(BH3bgW;|R{MOqv29@qX}tJ9fIb}^=d;`e!5n`>7yqqR zaVP#u!K$rg4e)kVxMnoXy4QpX4u3NM3D}V6fdiOm(!C|OB>W6YR<$W%m~jT^9Ly27 zmqM>dy)C}@9|j!AA$Z{+=z=xex)eGu!Q7)Or9FLr6=0hg+cJ(;9k4(Id5{Y@kN`{^ z^hN31HS$DPK?m~9E#VZ*p^blC%%LDT$dLy;#667eq08i#uc#+f_*xyti9Z7>28?bz z8!K77o;hN1b!6(jl#y%pH3mGNdn zdx=1D*#-HazcylVQfq%N8u}E4BC7}FOSB4U!rkhqVGgm&0G>y*El^Y|0VD%W z1t!0t&wzdlPb4)N>O4@5J!0%Wc9rg3o(IsYF5=gz6tSp#P z7NK(M72AUm*z$IvfL+TfOpGC4JH_x)@Pss;Q*Wi12hQk*1L%Lcfn2;J^fM96E@eby zIfCDby9MwbVpneGD3SZjXjdrsc?G>Y#95?;8aFt7Z(**v`6wR68pfs&XC=Mz&&<-C z4Dwm~8rk$7dApmLaX>rUm~miD8JH;qi>=E95|DGWT5~4&(^D@1lqxl0TGms!lA;A1 zgWB~w+(rr)T+DxCPbC%|VU`4fMv4$Q&8=?!z8Uz~5OaEtpv%t0*2O{I@AJ8f3Azpp zNBOcII~Ri0>u#}o&sEliaP@qy7NQEfml?obf>Zr-@o#Zh{zQrQsAmWT)Gd-vjm?s7 z9B6Zd8;80_vP~i167Ayfts(xO%|*AYS7TS^|C?0v?%; z&f}}}&%KbZ4r&9$s{^|a(lwyp67J&=?xE=bTCBsxO04BMV2_vbv=V~%qSX?zWjC2I^3{4J<0$9b!R%0PXd^iM%$((TJA}hApAFd^TCM*} zzx=h{Z0TzK-`Q%8w)5d=j)$A=c(XyH!Trd3sOwG4jnQp=o8LkT9rbdg&?RQs@TSu8 zM|^EMWqdD_aS9WEi_VKkxN z4h!1?iJlcj&%~d-$0Qba)b9-k9qT>(l6|I%Uk6tMbgUb^LX5-JTD0i*`W@@dmo*kh z`9G)VuivP3G8#N|EHMVCX6~z{Dq9aamW(w%vHnHe-oK}>mp1vw68-PbKU)tKFGBHu zhgzkWgpX>O_?+Zyl=>nurzO9r7sw?SvU2Q}TA4D4#a6jY^Av?uLrfdI6(^hu3#>@1 zR$$c^`p^tJ zt26YC%H4&fLNZ@(d7unO{wUXTX;V zfHierE_yaT>Z%enF+nw@+%pjo1H%En;t@aF20^!WVv!C|BiThdz|CxrGsw1iMs1!^ zn`iXMl4)p&n@~J@i5ZXkpz0lR8DN3*cy2SN+ zn@Fu?pr}{h*@;g_ZGbQ(b$39}Eo*eYg^U+WuUx}x#E^HdEc0sET#y;v5a;alTDwAd z8`JlQLkDtr1^H6nAiO?VtL}~8wf0kMKRpfmDWJq!GV)4Y*;DJ=Yc#J&{Xw|y-f8C= zewHD@6~3U9S4S&wIE0da+TcV;ggI2Z8Mvwe8oyzrw8O;rU0xImXWeb}#r^0Uk8wg` zYnpOAny2swu&2j{kpIX=Y#`;>1k7Yjr_b^ODJwpbBv;O^E|Yldo7IIq)-QS^^~@}xqugYuD@4T7 z?`i|Z!RW1@Sc{^6n;^3wb8MUx+#2>w3irKOGAS%GGo5{3Cc@K~-ojJ~_n}wF!_<*` zZ|lVB^?LmS`Oi(ScOw7!x0PyZos4_o|0>3J?xa7yQ@>#Aw`%=XPr+|h%A83uda23R z$dRv>y69HrQKAEmS188@dL|Lj$GNF=;XE~?K(o-adbA*a=vf1%naF27etXfCKUh5L zsMj-8K~^(=jhJAkgWUDIzfgsC4;*_3r1TFJYC<@+{&z0F^5^Vc6m81{2B z6Q{f8uTM?+>r&*2d1T$!cyU3vlyj*ELjY&Ed$xp_NHebk)+71o3OWvQLNm7m)`z<1 zg<^7=lL23URWgP{jSEA8vA}Mx2}|)=E63_Rt7zX@&$7&uMNbjStmsJ++}^~!%o5zjqR2>~@zCnJv6 zeu`tnN0yZ;skdds$r*OSvl1EE7XJDL;l+|$C&h;!&&a(w>WvI()Y8IX?+`z`IM*Il zm7&};kb=Svf|l3v6HYzU^*_p^=yV5-(6t8Zy=%y}y2CfD?w}PtSwhSTAZ?pcVo05z zH8E&^|6EN9vrB5rWCdTSRWx(zAL^w$uXhq$G19pHLF6@Z%SRVg$j*KyD&f$O&m79d zD9&srr4WLtrWV4_N-2i0b+f7=j(e*deq`lfW(HHMQrq5`aBD_O#KMFgGrXq_4}Gqo z5u@JNkglwx%~N5!R&BhZx&d8nwp0x7$r9^=IT=$Q`Aa5pe)a``R>DTn9c z;Ht4*5>q|P-JsPPz%>>$2RLFJlSQSVC>(o)_>$Mms3Bu%A5SubQhc@b-LD%)e8RhYQ)Vo8T0?}CXVS3g|^3^mLG#zSL#+2W5#wHgs_8d?Hlq4lrDISxQrjHt61gtm9%Ot9V)wNMu7$xKT47Q zRztQLvPeU=vbvSkzN{V?(j0urg$Z==-@CXGbbuZ~dMEm8NQ|e)(8}{xp11P6mFG`M zo(~N<2TFv!r}xMg(mM$A7?k~gOY&G}&4}?N_N@$WWq2#YTN(b8WcbLC#G&NZHG(@I zPJ}Qj$?qAZZv8`vu+L`1czg)O#}cg|Zw2`y3G&CO(a`$hTU}B{mmC_Bs{$eFBFK=+ z)ei1DV6jn+WkSRR-S}fX=QpV(|wsfHe^RB88$_K&^@E873Nt% zQ(m6Opj49O{`FE_|J&a;y#9_MhC?&rJvpd$*U*ahCnMgoe6w2wca@qynF>F3L!{e1 zNvpp~=&vI~(pFkXXSt+~{ua4+CSz>6d zI&0P0lT&9A_mrfP@I1J&(ofSy;Z-EOsPk(1l<{eJlp;&5-fs1FvECl^b88p)rMGvC z5R5_De=J9=vu5;i68koWqgBXHLLs*@yp`d;3?Ga7-?@CBivkX)y)!W)UQARs7q)RHv8dNX5p{Hb9 zlh-nEOzZ|{xch;uGncSNCmH9ELwSZFU(3?;WKLg#7pfOkQ$C+ZGuY&RnBsA$mF#f+ zpo@<;P8|oxO9ud_01`mghyW&mb-~3P5bFP8>h5Yp=w`75Yz=KBjdbxr=awKWudarG zpwIpn;4^kzFhjuhkf6a1pq{ehZRooGTbJ5*KOx>Z_{2dn& z3Ha3{|A{<2$H);~2QzWzVnn!gV$FN~!Ej`CtSgV+V@KF&(X&f9LvGs7)3K%;dR$$? zdm?Yk?O1P!lYHQdvEU9z|E0R9((mbvx-R6%6IjU?3aQj%|JIIwsjqJ3X-Z-L4;|}Q zgx1mVi6n^XH-418v?IN$Rs3N7N>shyGCNsSAw+Xl>Ru8>3mG5WE_YDHNVVLBpaF?i z9jcDxs=|mZ^i*Cg|XvmaP{+N)a9{8E7$tILhK?(t$}R1hFmP+RpR^ZPkF~?F+u#EEvoD9SnticFOx1xoj`hKM*~NO zgz7ktT_NJABbeQjFd86#=V0x>@nusWjyM(jb0J-TJfA~=pf5a>>P$p zbWnqz&7;ZVY!jXf#MP-g^4|i1pR_7q;Jo8hI`4Pp-o(f=>zRIkyB{yyxEKDfwlnUD zvemI6%Pk}Tx$ww^yZF-MV47l}5a52fCXROjt3()u{2|#w#iIa%KD*lk-yiLB$u0Q| zF_-t^N%u8y5Of)sp}F`Wl%?47v)~x4Wu_<1Ov9cb!A6nDd+6Q0rQV12!h=q|SX~{U zyt66QUad%O2uc=z?Czl`T2%KQl9C?w-st$aF7>w25|^Xlz!1}}mM+Y(+SOB(1XS!S z3fq_W7bUwW=`zanQ$sst#+Ij_QdsJIM`07a`)DRG^?gTS({JED%1m9Z2T9lZTRD+h zug^CXIYUDTc8WoEcl@M5HRJn9L*1wQM>B#_!ll>=6bYGs0zZ%xH?0q-^#R2=cw`Le z77KTcR`h-syIKKg%$`=@nz5rp)bc_pW1V{PKYnV2a>&d*x(dV6zy${mMN&g^H^uN;B818KxhO|Jah3qiba6AF0kQ- zArNtk5Dm}`6|B*0kXE}1c=G&wXp>kR9vFM_;pVaBcjI7cx$|-~9OuSaPn&TKPxF(G z92W&X7oI%Ild2pjf3sTiEubFw17dzIxlkq@v0K6^;2vHqkQZhqA?Uf=)^Po?*l>~? zKostccHqGfMq>;ZLTrZn;s_S8UetBC*Gz0pgH0!RA-08NA=DkG07g4WN_>aV9D9tv z?1-<)ErA!11|TEm%eL0 zAN6uWmqJqB{X8cPs+segG}L{b=V(S5P)ZCGpXUUgqKMVo6P@-%r!)`ub0enMC6gW= zA^{?kK^_`^40A)GBwbvPsq{qb$`b(`)8_Kyr_g7@(JtH)bo3}(6C0IzM=Cie=IvQS zk7Ik|LNE04G_+kzOk&Mj1=99fO_Fa}{%NxOC^saDU8DIHMKFkePVnQoycH(XYp{##c zR;BXG!}&{0=ClOIt8NNwPRX)XHTXOnMSj)pcZe&GvBHMV0R#(t9~PSevn{wd|AAb+ zxz(M2zFk?nh!_loJ1SIniX_potbW2p2eP<~bmyGW)#^N``y_r`2S3!!geTU&h}--3 z^!3sv|5&2`{rP80Rcm2EyBloZDY5uZS!~IFoT_a!MXFj#UYjtiRmONnC?y_TBkoCo zJQ{SY8yX}6`SW`T8Gqw>*#c0PG=!k;pOJ%qn{=#Go8xe2 z#j5&gd>*S79P-I=zp?${vXz!4#)EcloqVII6R}mfLR@GE>h#6MS4$q{h(BhSKyUko*i`z(PplbaXgrFc z8GRj!ta6rPD4q6TX!=sPN+wSE+FHJBf216nWSui^qQ*cuJ+mRlG@q^nnNB@blPb?` zCo#?qlV(%jl-gx0p+J~@1u;+7CrnB^6s!$E7WfKMaBFQKLfp+e2mcqqWKPB2p((ey zW(Zm@Lbgq*>ZZP_d(t2D3Y-EBu>qVPNOOO1W zJ#?=jap;O5#yVhq2!arY!ay+A;PqSDS%_DS78nck_I?dTVunjHn6f>LcTh-M=0@_w zq}DAHPv!tQS)4T()Ez3ihYnhy}pS4aWo z{%NFxzBG1foSl>k8&T_MJntV*CbI)HANCGL&JjA84Uv6t49A0``Dg|Y;l%1#AFeOM z1I)lbIvRAW_wY+_O{n~R2FWyj2Mr|RHL7(xO9)Hnu#F(=-unLi%sEU9+cKM8HR_nN zIs-9(eZ&&^j=#w+H^B*IFMWnP!M*N7qSUf^;pCsY3S z1CtM`9?V97OP$MxNS0}xz`NA_(_$=T8b&ivihfwRmhBaoJLsgoDI-U5`fJ4asq0dE z7qt`uDZYAFiFkp<&33J><%_lvzB_doYkviQ`SM0?q+fjeq*HJ?haM^9^2p3!QPM`j)sTFN5jc*JlTuqF-y87t#H3YM3V-%j;wW+ zIqR87yX0p5c*e`~ijqGZn3BZQ8k{5Bg&tD;yo+i{04mt{T%4!$iF`23(>^0V{I+O+ zQzF|_%TbWcoG7!?JQFrJHf3Riv~ag07u+@DBddR=AaH2tG{GOvqn#tP zPOTrrsrCH~qdjG~9F2SVsW8__2$WwCboow6193Rzj$-{^>J#;Ov4D(s(~zJ{Vg4vd zZBNPdQ7E%gi&VCT8;4wZ5}FBZ?^v1|X%%&asoqq8Ya2ViW9I?;qYTolY=n(~q8$2z?%#_39IubVlp?1N4iv*NiH}7eb%vNEtTaH{;T|r8yg8-*uEb0vfpJh`9uCLA- zEDg{W$#{HTI5Z?5C}Q{roz=U5A^O~WT(;R@r~BbUc($u#Z1`oKDG`A$I6Euoqtfmw zF{Zs8q!nPSJy$FKri^uh<`d`m7!D4`lkwrfs6RPAn9S$!;CPOI=Dqo(H%17>ibDD! z*QWB2{a(+Haj}lZy-9xtdk5qG%sChx!o!1^-J2Z1dGB!Ew+CZ%=tSkN3n1P=b|<6- zg6^yP-YdaVvh?>WVSik2$R7}M>!c@rRUWratU+(kJLvZh`h%O^@yTFt(mVRM)vNj8IoFSWI?*`ODp$Z%A2eHxdIi5+D1iNO$68?$`<0Fxiq_}Jn5$VoS}fQh zq=xdf)}Q9!LSp`E&8fG7f~+0L(E-ORq$_E!n50JC{pYs39vf0elx&=G$am3i^7}Fh zE-jShhs6p`HY=^+w4Kz@aKWe^LB(lNE>Ll%Z<AVo$zW@6qaNxnF}{6LXxQ+hd0lqC@eY5FnG*q*9VI~rII51$_X Re*gdg|NpH_;jt^$0RWeeu=M}{ delta 10184 zcmV;(CpXxNQHD{F909hG9T|VeYR3Ql9RR!&;7yccJKXKG76~340Oy8-a}K^)bcKkw zvaMsQ-yL)htd51rj9SOvERMO0tYhnf@-+ijm#5(5^4#iJcgSOy68S~H-yQBb=+1iR zSWCng)M3`~=WiB)OY!M@>>`_D(mk;`^-dheLyTD+%R$Wc@QN!ifBt{@=bYY<*_znk z6$3v_IQ8HheZ&NL6Yi~T{u;yqyjt~uO$kT$ynCwt`x?B0>Sfn&)B(uzsP`ItyCF9m z&j9P56OKFr-2^t^=br(=u4;C}J;c`@0Y4G+H4X7~2fXvV-zcA~S1amq|+ODN!>d$?S=Nc3aPdi}w0WPRy~&m|hrz4Yh`c^r!t zGw3qZvG`_%L~}a*FS5Dy+t<16TOX%A@z2|I88T&guWV815hTtcQnAKGmL5GQ;z&X;*>KQ*fe5S^8D2 zbEr7ct~L#ac<+CNlJn;{{kE8jzgd?aI;G1c=3;t1>RQKS?YbRH`|uihOH3d~spn@1 zx|i#zi`gRm1~WKy(eGag^4>$&okIIIpnLz_AJn@|zkDcSt5xjLw=}ZfEX>hTzA3?* z0c$>l51{A4jrgq_S98F%Q}ZooH>K1?ki=T1CrZ~eyPSWG;hSCjhCn~OFey3i=Say3 zlK;kkqxvE;f}xR*9kA+yv_=N6;1qYG?+S%x)HF};Cty|=nTc@-ytkH_;iRwnPhZPx zO_juyX2+0zZ5Tpel9Kguio6V%-{fO}8Gs;c5*zk#G~7Qr9FB(v<1f+R{Fcv~IgbW< z<~&^>`}Tj*qjy5m)?&>B1S4ZkarX+%vA_>Gmvq7}JxXUCuo{5RUzh^xFYy%;&xb&W z_t)Un60SZ+>#vsZ3*#Opxi(M&tsG{~0bhx;)6*b^ErutCvRnSvPwmc_q&{mzqZnj* z)eu1`NhQp!FP%Uke=g9-N%QIatWbp~Kag~njg+-fd5MOZB@+BtYE1<67y^NrR$n{h@Ms3h{I+Gb2K~X%I<{;`{)Yo-OMHQc zpaWcL-&z8}736~JFB8Pk{V8Ue`W?PB8}FbuG^i1}T&+0@Bu@=35(8jTBA>rZ*VyHl z_=|sgPG(eU5V77u@LfUT&?QlpU(`CLZgTWR41pYVp@CXWO(e$|Xo;O_2;IExT)0k< zY(!_}6UH<3o$>iOCE>Et10f^jTkCLU3mUqy@a?`^Q2b=;% zmk6vNUvvN@j_m7(h|a{b7?NKXhyWY9uIMo_Ow1$(m`EH2L&$G$NJ9#$f_dh*1$WS0 zBP+&kK8*Te?YFh-YJ{tN7~4FBg4KU@pE+|6_Vvtt2IpcghCZCuV(sX2{VB%hFF}jR z0(#DsT!%c%ECQHomua9@vdj{A>v=zhiMJ*U%%L}ha|CSay2uuD1gPSXvy%@%Htkko z61}?Bj7+0&^{sZRa^HzX(`)~{%zeLh*+wy=aWUi=?GpAHZ5S?-)*j-h{x|5mHG z75}AR)z-2Gc-tymGn!`IYeEHwzZrl8tV#610ZcUM-jEv-eg-A0+LSQNI0JMBW{BGh zp;x5d7GL}i0}kX6ymAn9!3u6%3Z2(r=Fz3np1!{du=SK}7)Q$vm?MHb$ORlo045H) zS2}lvJkeFqfqZjAI0Z9kV;6sOC`b-+IoITQipNk&wz>nqifGb zOpfe&ff*2tsYZqwz+&g*AV+*+Q^D)fHc-9GX#h(+SM39%B=`+H-<>G76ml}ajCugw zA#Wol_pdbqD~xOePS4Le$~bZ1-A2YCKXts96h`vH^3DzrRyxGuR zAdp;kK|biOwOE|g+KYdNK1HF3Y!H)#6H>$iEkl}cH#%yVLF_VsXAx}+6xB)q$v_i< z$#3W@VB(MN4LRpv>Qeibfi>aSO;6Aa6U-Kp_F4<@8LOlHalSgbz>G@*sTY%#1yjl* zRBpXudoTi9-YyidYgvVfG30Bf7+wmVkj69WEfw>?DP3~_T{nM_i?@V+C4$+djEF2p z@H=rg0Nz3D%IzE_a-SLP3I#tep?8Zoi?mSV2B+^W%r!S3#iLlm*c9Tdq*wlhS(*n& z`K*0~Yh?aj#1Bx(E^S^ z?fM;VB83aiXR&{$5{r&7O9DY7MTnf{RyRLy1|Az?PR|f@*{RsNILP~bHghpSSApRu zU)E#iLa=(>Eq3p@%DND)p3l`nRAKir1K3G$s(&v2Jr2vCD)Al}LIHJ)=;bK+}eL9m_hMT0&x8y7pG)s174G`PO$?#L0b$FE)QstJKIxjrkKptoD&}8!XXFX z5Ly?UbATC`QE!PHFx>zM{;AUSI{<=H>L44u`ntgO0$}1`TSm7*uw1(w+v4rs3fi~G z340Oj@mii%Lhw9RpV&!wbmxrs}vpH zJJK(aF`HH{g=TJ7sDRoUS6z7oDv#6c`tte z+!TY9K7ikXL-gpkEb?+#q@Nd=k|BlW6-b$5{`%~(Q}v-ky=9G;!y1FU*q0a~G#yP) z2)ZE_sS;FBpG}NIXj$duu*xtm>Lx}E&BiSh0B(p)%7-u1XA%SYEsMMy78&Km^-K_N z(9^7gW-fddIEITkqNKyQIveP)-8p~lw@&3}WCdb&QLl#6WY56*_aQu=?#hrjRt z{`KGgxWS8v{o!Qa zfguP>vFHcHPuT4i@=gKJAD{o^*UnMJ3%V0LC!tw=!fx{p)#JeRl8j)t%JF#$+npSm z7mPPE%f;=>)G-z)rhd3#9y?6ZF?_)H)sw9y*p715`8j%~F-E2OUes8XsH#B5v>BleY_- z{9}Rs_vfFjhl&@W_(QEyOu|RCOngpqHcEYwnA4J9)C=U23t2gKORaxQ8N_0%T&8)7 z!m1&r4c>|qPK5p5$Qu_4AI6r16})7I)D4#}W5!U|cwWGt&J z;-H(=8Y!fAqdqjl&gu+(qjGm)sgTUK8y+ZyPlU<(-I1EOgqtNIT+YB9bcI;YJUl11 zhaf{z=2w&88Ste7U`>DBmy4c_kGiS^O-xWtDfcoKaHfUkJO&$dC(t({n;1Jp=% zkq&S(+v5zfZJtq^XVm5yJ+fpP7~&=rk6vKL<36Z*hg=3&=1PZAqtgW>1mz|p8K4&w zJj`@aDWKduq8&w)YY^5J9MrYiZ4T30$m(VCoHJEym?tV0#x8%XJZBR`{M?=F+mDDb zIM{!3^1@bNt(w9P8V3EO7EZts14VVY!Qh~kT+>N!yt zKJYC<_l(d?+9i2nC1I6UHK{&wm5!mKni?oOx*h2v1KQ?$a5XJ=tEEmH=lz;E>=ZlI z<@HHWrT#tjLYKIHZxgAt3>5Y1nVtA_)CLGsQg;Ug-Lij1_j|~A!Su>Cyh03lcgixa zhRr#d(lv2TPp-5pl(#Ych&Xg0hnJ8q^bNx6leOyJ_+4v1wf57~u%7}-tR*8a)s;Q9 zzP&=Tiqs#3>+YO(uHaW05?tX6N_lm(0*6B=sSQqqM3_Uhn}Mqupz#|&hZ#0B(|a{$D?@)e*k-W-w^U2*@z9K9Gifdtm*VwejsJVN0Q{q+0|td zkA1Vcu*aGP_Tv-;T6<6#EeyzEc-u4`7A}@F?dk952bSHs3p|E+w zL0d1(5<19DhPp&VEd91NP#lcj`iZqDx(PB1GRMYA!L4D>q;TJfC6mH3Gt=4kWgf}S;C znu&bYhDW)XiIZ*f*C(d@bt&@1JhJXvJU=H~%DL2oA%IidJzYRdq?y+NtC4(k z2^|MHp_$tOt9{+`LNPhX$$+ma8N;E*g`vP$V7JqRrTDCsWA&a@wC}8ES?0;2r-*-L zR`etZZfD|N=Jq!r@<-YMq}X7GqZkev7{vaw9Ym`_E=Ik+p)z|Y78^=#pA;syOQoVq z;Vz>HinH5mXoM8PJFoQC=C8l>{PkAkT#R}HQ?fiJn9Sdb)vqOI!p{wrGn5T;jaL)I zbB|O)K+V+2h~u@J;u!IfWu;2$Z5e-Ya*CbstVBk(g};75c)sA)aq;2D3vzFcdP74R zwX|^92gFa$&$NeCWhgfdq@b{apyiePgi{Z7{g3h}I^97dbgjX9=NhuD?(iL}J7`5u zmJqW7NZY2A7*gkFO$^#USChi*g4!}!!53;3&7As&dg;#Vodj2mG_HRTd4+%6^3g>V zva?@^N*EdPnM1i4#hGoT6hbi7)I#`KDa8=BZdNtKac`BwkE|Tb%wTF&YTFwVZp~2 z*Y(30?rNpU!1gzg{IyG5P?(OC3hde%zv)g(+( zGjN#71hsmH>i>J1e+B=yDzc;6q=g&T5-}LXe7l7A=L%r|pkpmDiTz4%6(@BP^mZ#l zwy#y57f+R7qOt@MyNQ#L6dQlzUMzz;S){t^A%X-~>Q)qE#&#Qou!U|tGUO2`UGN@p z89!K8vC8i&Y1LXfRCd#h0u3;Jlp_7DhHN!tk%nw#bt|ittnT*=X%4>R!UVeb?`_-& zIzW#gy%YU4B*xQYXyth;&s%xk%JZir&j*H_10}*f(mUh}=`DnL49b7*C3&o~X2f_B z`&NdxGQ5@HtqgxkGJI%A;!yJI3c;-pCqfvNSl_#NA zTAAL;bYG_L8?vL644Z!-=$_K$67#H}DKF1sP%6oC|9Yve|LyNWL99^6>zr%9vmDiU7Qd9{4X z_%u99k)>8|w|cvT-ag8$UEr7A-ZDZk24(lL9I?)t(aTBf+Zc{kAwLO)+{*A)hASC9 z=$Ufl*M#FGnyja6!z7cy7Ak*04xonsB5>-8-Z24uff>MraY$^`-MSD|YRH*I%f2sW z`p=fS-oR4kbMTjtnNbdxx=@8#U!IQl>4|b&-;n9-&+W;Sdf5ve$}n3TKv`QU$0Uk_ z+twONp|eMEChn!%Wn^P zNk%ILaRLycIgMqVNu7sjizyMS4fuI;??II5_4moP9zTEe_U^TWx8LX1;N=Q_* z3{)u9+uA%)MjIK@C6#0isuy0<6SAqvYZ*8uc7s#g{X|xoOIV|mj5ElgJj0N$Woddc zqp!g$)r+brpU``D;H|^)?SQ8FCt}fv{mbc}0tarpoKJdj@aEGJ+Qe9N(_jF2K7jonY ztmF%YRO+#RYsY`YSGV#srLg~pj`b@-tLXSd5=8YIKgwR(kzUm*elUL}s@`vzovf-5 zqB$#dFNvasj1O*`JE&r$TJA#7fJCbfRY!7FVZ^Df)P*Zr4yf8|l6k|~53D`yRV`_L zsX&d@XrkJb=uk~^X?A#>gg||qLr|S<9JgXJdLs3*iobv1D7wT)2s`~vumDcHn zSm*nO1avY;|&og9y8 zr<_I(FPF%{kRx{k7$OdnH))9XV5tsql*t`f#Be_M&|J`qyc0n+?cR_p-;FK|D1c|t z$B>c#;R1g&-`Zbpo-WOmCOJ)OL}pfz$MS$qr*+>$>>@|4fo!^hTrA;b;`{DTdBNt;GBI2kcnC+yL562{@%HFA>0hueEYG1CR z$Guo41xN*lauB$faX@Ea<-hS|Qy`8w75j4`U4VZ)pF@D4dmc%tKn^t^dvdRB$-LgJ zjCr)%T3#5>9vYHJ)xv#1{N4Qu*<1)|=$@@T=#xfW?lnNv^D)Ou)D4;xv#|ko4nrq8 zsKL+X(d2Qq3C}s=>eLwjMk=ziJEZFTsQM9P;JR~JO?48l^QC)xPZKEYFM#G*Vrd=&vm}9lArzi=i*jW^| zFYhl(c2Ux0l4wj!F`mOx?B&EuJ!kFBDG$h z?<#Woh7jx&gKY2kNrP&}_mhUYOZSgv1f_&au@fi~G6jAhDQ;RHQ0oJVaqz$x(k*`$ z?g}mG-8Oc$0??Q}t-v*7M~A58g;K^k_2hs2)ClE}nR#>>hNXdX4jhW4hUUmyVgk9` za8j~oFCZozpl)NgA-N$_1RP3ydqDMb9emUUvP;3+kT`(Q2%vc8;yVqnJHQj9FK}jZyX?;PRGX=^E_@i!aXiErasUpYN!J^e zcK#AMV7dVi{1bUh#=SsrN*!c_SIYT@iGyt^EFoB~U5;(>c5emkTjYej2-bZqoo^6) z6s;DJExJjG;(MzDF87a@o;RN#<%TYWq`dukP8w7*=Q(MpyFAa)jG!5m7$`o^2|Ps+ ztG6dQ?TJoV9v<~_Bc|8|lOi4>OFZ-)u|~P+H@(rSZBnADRevz-^arE1U{w(>^m9X^ zBwd`7iS$Hl%M$?{)8_K?m(XXz(Kg%?bo3}(6C0IzM=Cie=IvQQk7Il7laC%4e;_v` ziCv-DR;(_**Q>uJTzytV^+k{=bdhs~W+j+ffXrcLJ={7B?c=_6tgw#s^;l+R+Ssg2 z&iT=}*XzZy&(F_fnY(xf`e?j=c+l%uLHQ8*`#}CA#CNo-s<5f3>HK$}*s~UVBjv~Km_Xot4 z$5>%QX8?jZz6*;@f$0XEpZ!3t-rVX=->;acs+I7JqrG#>@MJe^BQ$f3d258lT6i1&4fc+-+=sxNK#nJx)Po+NX%7>I8&T zugRB?gkR6#`4tmKpMzm6Z!g5b5-|>!t8yd1_Mn|xC*Nr5L~K>A5Et5lI(>2R&5}ns z;*TjN(A&HtHkE$yV`~Z-IvB;!jJ^&ZZP_d)y!N3Y-EBu{A~C}y8BEz8#ycpaEpsh-VqEK%i6?V_oGi{73~GnP zDT_)^8z_H=b+kvl;kb!X+S$U@&$1%;$l&sdi#=ls!Jl0IY*_t^sXg$IuaGpAlDM8kWqm7ofx1jAKQ~`PFW8D%|FZ*bU<^zQD6;gn?e;Vna`^HX3w#=qie~miktj<78AF)Kfjcl zOP}Ei+L&+Tf%nNB#4el)%#S+O`$_Wm6=KM{6Sy47$&~;7#N>mj2eT32Qs=TEl4V*a z@Gf=#v=~d7hS3a^q90bSWqSo?eVw%5GiBr`PJfFSKXF}ZZ=;q%AjMbjDiJTRxZbSP zwS3Vwf5LaC4rA>vAzxg}jr6O}UvvsC=g=dCTz(nI4Crd-9@&W~zNfG83xAW7yE}K+ zPPSu7GT0gN!_jd6=x{h59*lS5dCZb-Nh{nh5z(Xpt|Mz*WzKpg(l)tSKc4aOvZCbo z`=%r@wFYO%cAXNK5Ke+gCa8M56H(yXAW zS`P3!VlB74S|fF3w5S@2%b41h5gw)FhH=**Ytf|4>f^yqs?R)HZp(`|JoomxMLQ|tK*qdjG~80{bB zr@~wzAy9rn(B)et4aDJ;JBsyxsZZ4B#R4+kO+$h*h54f-wLK-*N1@D0EmGMUZX9yu zNoXdty<=%=q*c@vrg~EWu5Il6f0nWHTUX*~MfSz$pqHQF<>!^u7S-bWG9KzgB1?%) zpJnNz>2!ion|}0?MCi-dtRM9AGlH(Y&9?h>AH=S=?60PjmT_kL z=;#HYT9@h~FB)UJk8k(!?LIzIf__ZO_Xt#Zsv>t_$d>todbh|srGy9uf2bpI6A)?_ zyhuRVcJrP_$!rxSyXDAr)fJ?KItXwo#-iR(@KrWt;Og?U!O{S2k&MUZg`pw&KoP?~ z=(OGi4AJN2u`9b1FmptslW@AU`Qy`$s7;JA1AZ>wYdLK*j`e|J#>W_U3+#+F2tUEvp%Bb?(ZciEsnIvkCM2cvPvI=%eF ztmA`G$NC`u`fR}um!Cj?I6Snz2zGr(;M9#+^CMb*V#xW3c8OGi!B2QEe|}i4m7mSW z`c(e5j(ZOU%CAa9tPiu)RVo4Uy7<@-S#9vFze0^Eg!aWSi2K1_3-fJ;r|B!0RR85YB1+3)&T$p C-{o%r diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 06d91c92acef746dc35147e130020f271601ca56..84b1b1b1da844a925216e4850a5f3d95f5e9c7e5 100644 GIT binary patch literal 2710 zcmV;H3TgEpiwFP!00000|Lk2~Q{y-i|0*io3v8MEhk;b>!yN4HQ8hEm!Orc&Kpl$R zNwnCKSCWAYmG6Ef+llSiPOOm3h2Wa143^bWx72@rYD@MD<~}gt+qjS0jZWhPTbQyT z$Nd+qsB!`K@h7eViLS5C(Z{PxY~e=`l5!SOwA+pDkpqu-YGDu5m^%{pzrA2aJWcv5 zb;0J8H9k7d6$Klxg&mN#puUQlxWB(2@mn?wn2io3`sq(D9zfhE5!mVw&BzY?7P2{0 z@T?l=(f=0m+ej)d$OsxYU@I=JVQ3-0gTCF70d0I`K5-m~8n~g~!9so)mEGyh+qxi% z9HIk#Nr%LS#yN4_%L@yE5L_Ihms@tL=n%=qB~u_6ap%&ZpMOS-x>1!|B|rs&p`RE$ z<(fOR&@V!K;p!&veJ&I@nr5nhVzPqv5B!!ETMMa$(QF^yvgs|mzrV*8mVxIHF~NPD z^cIb$?o-|}Y+;`W;zh$72t7rE$&t?`4W}c%otZ+votv}VsAu}t7xoxDqIC$KiRZfz zwRm}f~mt#2PBi84FzZ&cakdtQ0v3vVBEkg{VXXEeQE-Msug< zc2-7&AQN*bySbhXWn4#&)VQ=;o!(u6WQJn&N=-{Up=wrDZYi39N*vwkN-}VjZad~} z(O(sEx=^8t4~TrmQa_-L8Mi(}|BecNX8-V7ULscJmrAO;(mQzM`7&y-ePUscveZ>v zInwc|m6Cn;Jk{E+Cts#5B??>@+%6dD->4b~10=xX3Xn_L1GaD=T-?WFrF_|MHb-HV zhRwa^FoqCEzTld#rmNvdbKvqp(<4%WXcC_`eZik6{~ZBqjQBr3^*)@mv`l#*@(Uva z7aT(z5>pkxkVAAZ;M`q?4r1bBa1f1{0TxJDDGvPxF}}#|wJ^ECi3?ZTx7L!BU(|F! z*c(@+5A9-A`W2KtHdLgL#th9Sjy5XS{cAuG!rA2J6;b23xR)yaXoti8nLA(zYC~4H zg=lBvPz}!3xb+?4)-t`{9;n*@qRRu*yoFZMIxK;>l#OtTDXBzQvKg3}^AFYd^7tdH z%~XvvYoxhvNb|H)0uLx7F8vPiTo~en46$kYYgT?u*vL5SXL{aR{9lxB1_WvVuoFhQ z3jkn2kJn3B84T_3!=VIKZEVBr(wYOaX8Nm+optQox7c~sEsmX+Y{(5GrY0%Q&n!L+ z(KVMvKz+Yb+)J7>ihVJsTS?y(F}K7iWGYp04FBR%&AZaTA3YK`(7yR{#JGTKkUS9f zEd6ZZRKW}W4BpjvBB@PW-0yWPU2yY6{Q1Q-NDz-&M|HvZ)mIt)^;`ItcJ5O{Z;~+l zGBc)l#WF46d0uChxtcCWV9;~KeHD-JTU>MQ^Ky(X`l!r6R_P)$wiOAYnW^1W@3hq` zc2l4EBiuw7#n4-&C^wT0G(`ES{sH`*5qBkdja;SeCO6IryMY)j z<{+-VsaYz%8+nBsCjG>1;)c8z-Tf2jXk00ovcVL|uG(7Xq{P+)w?WDqkTOy(Bcsa@ z7h7KtP^avAlZZD@(@&08}wxJ5pTEiWR}!9@_}a1ZH@5FetaP--M2dJBz!fzUG? zS9B-nX$&l`cgy^!KO_|q%52kPkx9>m&P`AHPK5_&l>+uwHruTZ!TVum-4DfPb`35N z31686V!rohf%->2i&^M;TPwA^g!O7|r?BTPtk&wMG4+zLUJ~y0lCXDL7C*yHKc-<* z6O~0vGN7^Fzq~k%B4BX<)QuF!J(Ktrv!c5$4r#MoXcPUe#u`O#D8{AbtZ3~eHg0CN z+qD2geJ}ErUjjG3&p0kjCH&@S)^*>waUGu(8<$>{pL5TnDz81uEi6T821+BPB<^*% zeqtcy{AgTgGsIT(5pp$p@xi+Acun*W)jwY7;+#6t5ag!B*cQOej@06wV=YDvbax`6 z?gG%wTgM(Al`FOLH!W*^qw|8t+WM*`?CCttn8t$=I%KBMr2|aSAx&#?k}&;0tYK0k zcf7)BX}@04Yt7Rv{TJNS4wDQgoxyiv1L;0VXM8g0lJ1Ebbmaqh{$Kq0Z$FrD&;M)S z4Pi3u_J*_@OiqFTdY#8!vD+8S)5Bmwxk->!FG%YJ>08FkA4(2!CU`C% zzN=#9j`_YDT^hs4-vhNa%Q*^6j7sycZ4F&N)DoD zBH}s==7e~?HFFY<*IPCx{r?cqP%oZ$&Qx~+FcfO@+rtcX?8)^)}YO zen2oS+eUd>ChwMK_WGVnN<*pAiCF_Sw{nX;R>lS(LYoo z2})5mx@8`jAVnSk5j2D+B*Es)kw|f*-8&f(1^Pk}`GB;#JAu2wJ%&H#?VR-{zE8cg zPRXIkc>Cx_#|clxnHo-(x_pr(C>t%apKLp?HliF`<(9Ou6;YTIs`i4!^!rCP`9i{X zr`6q(cR%`dW(C*`sb33y+F@2rGoUV8Y z>=FN4e&X+BNc}QhwlJQzyku#-qIshQb6S#OLzvY-N4G(%)ddv2xZu?b;T2NkW@L5! zDhaolU1=?ynO6TL-16!Zr3d9IlW%JLv@$`p=c%KUl4Cj1_T6>9S(Tz(_tjpr583>4 zqTrG_@Vw&FRfXF%j*(5B#yNEipSYy>-`3-n>=4!Gaq8zTm0lB;Gm9;L->KK?o^Ik3 zPkZ~Nsy*93lRp3xd(YJ~Q@F_`+txXCkkH*rU`mT3!5tn&bq17;qA0->B90?KN^lUJ zx7I+HN)N!03lAJLm>`1w6McixLIj<22W)ikG^X|#QRYxBqVB^BTt#ht_sA#q132?u zRJa-;%s~Ww*0sh&jvHCU+00>^`a==*Q_A`&Wdt=(OHQ-IwrmJ9W2)jM+T1;q!@Ri4 z5{MVqFhX_-S6g|%UfED3?UXq3YiO1!bs`bGkyMEKPpR z$zGfLO7~x^eo0pYm)j2#1x!&FsWAvZL;zyZQ#y%6z*zWf@0A++&XQ5btyTw5r+3r; Q1pom5|M(5c#94X(06mH~Z2$lO literal 2709 zcmV;G3TpKqiwFP!00000|Lk4eZ{s!+|5pg!OHwHQ&X<8c>?Lh4!0tAebgvJa;6h7d zn~g-OB$dPs{J$S4$(CeEw&>VB#}O9rMihsfA?M@A5h=Z5?gJCPjr+LW=rm5Sg((|y z+<(Q2Di?4cf95KX=;rzYeY(EF7JdRDDQ6)?yWQv>Iq-z17WP1mxg&A^`zvO|)1*IB z7i>;hZ_=Uhlhs|zhlFI+2}x`AAaZJ5yY($fvpbFjO@T~A)7M= zFRF17{cj<^jilm&jG%D~w&LOjh8FTW==(hx(8f3B6UTw5fm`|)Sjg|9vOB$bTbD$U zLv)}o>5$maxFD{3b!j0Gf{R1+ddKb*9U|GdVhRK!?p!+bz@{1gJnT^aF$E zTyuvO`bmhdT-^q~&xHa<(@gbGOjgkTf#1?%Yaz8Tn(f0oHoapH4-eSFGVnYiCb*B2 z-lFl;eabtAE$kCPyl8j>p{HmtIr6!r;dI2eGgIidb90s(^-SOT!XAT1v<|@w@q8De z7OyUK(HF`<;C{Q+YFYSe;0^TE`P9PKg7e{T6CZH@YuJK`>blc;2_O6cw!&GV3>x@8YvF!sVF7;zRDgr~ zN(7)M!xvx^rAtELrdBdro!o4dSVN^bV}a}5TbL?{l_DoWwy$Wc5S56a1tGuBXzumg z&dP`oWMVF5H`lYFjO)me8kcsf)4MN_%utM8scC5^RL!c&Ek!d>iKBa6Nd~UcZO6PV z`l~`t7b;Zo0g=yG>IbwjZ7OI^j4 zBORYwDcN_=Q?2cK@@3jmqQG^@?Shg1g{pBdKmt6j0J)SsVhabt#eFVF|>gY=l!xNhQLP&A`l@f2hWn$Dd$r zrfQ^FBh7t7nkVNa@PIPn(!U^{3qzccAvR5a&B|{G8yScFOwU`3|BLd?fItlZcEU(^ z0RSxM@p=g>gQ5L(IFz8Ojcu4+T6195On=p}vyPqn7CX;c#j*2>4Y^^&)Fj3EnZ<`8 zy5_P7sP9*bdr5Ofu`lLyE9sje=9V~xOr?-Q>mvVYd*Y z#T>-dH#JMecO$Qm!=#_NP27<8qPxEX9gQm`Q#P0)*;QL>cS~$da2uq&1t}xtGBUah zak2FU0d*SRzAeXs*@l+b-n=z4gInaY*zzJm9b5!q4)>7m2=O6$4y8siqPNfp7zjPX zaYc86p2xuAdUwo^`a@C?q0BZt7Mb*1=-l+A?^JkjRw-a_WwYJt5WF8|*69_S*$ucr zBz$8Mi22^11?s=^SU)u|{2I9Vea3NND&aRrv#$HjjjMN3Y+QO(e!)GDs=W3rx3CnY87PgElDIeF z`iX&*^P_R0%@AABC&<<4#Ru!c<2BJkRR4IPiwo*VLy(&iV_N_>J5q~#jPBSU~&=!(Ca+)qBX!K+cR{PN#|zsnH~lc%1wf-dO=z*NZ&JNJ}x=Lnc%s6 z_^yhXJLdatbZHDDe+|^yFt@XPnA@pyxVvVXy8vzTO8oxdULEJ^IJa+c?xf^|xpwH+ z0@hCtCBz%a5G|~x)`)-!L)#338L`vpo;nupg#GS9EZqAvPo4UxQACX*J`jpHEjfsu ziHPegm=ogl*33ybUT@i)^#8YjhI;Y5bEdirfT2*E-ydeEBVQf)_AT&D~zPX;od#zGC=Cu_#REBRAtar^ zpcG}JJLZuIQsfa3K|^>(5^T;Ki4;fLy^|49pf41Wk4US#6Sy1PWBB*HowMG=_qli8 zE;%$AZy){SIN_-{Q^UzpmoKseWus;GlWphCMwDZ#+>$o7A_{Xt)n1U8e*efOUr6}j zw7OGrJn=UoA1@={XQrb(Xd5YjjS`>aYK7KDC;5 zxYTTRY7KvjhIdO&2I@J95WpG=9uaO#zoX_hq26=GYVH0I+TANTw=57-Y z9`V2DC;ncB)UVTJ3*&jqOP0nfnm1Z7rzI&igjo%AbQiQ*T|m)`3*NjCULi$pMpoCa zl5m^ZmDbXkY4u;jEw4UNdQh%2`KHDXD-%@vojN){E;*JHZQotzn^h^wbzki@`;g5) zCkn2Z11~E+T~)YU;~3f0XZ2MOK11g5ko65QcYRA)fhD2ftHA>ud!qyz`i zd20=HrSt#{x$wY2g9#$&KhZZBEkw`-cfdvm&tqzj5oHe5BI-W8z*W@NcaMBxKY}yw zMTM&o!W=}<7hP*iy-^vQof}q+HH3`XKTmN9aYHFl5OX|6RQ3lP*8>{Ij1XB%F^W5 zob0W+uXO*#>X&piaJl^`QNR>+ks5;lL Date: Thu, 26 Aug 2021 17:36:06 +0200 Subject: [PATCH 9/9] fix events API timeout handling for nil blocks (#7184) --- chain/events/events_called.go | 14 ++-- chain/events/events_test.go | 124 ++++++++++++++++++---------------- 2 files changed, 73 insertions(+), 65 deletions(-) diff --git a/chain/events/events_called.go b/chain/events/events_called.go index 026ad8a4e..ffca57d5b 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -145,7 +145,7 @@ func (e *hcEventsObserver) Apply(ctx context.Context, from, to *types.TipSet) er // Apply any queued events and timeouts that were targeted at the // current chain height e.applyWithConfidence(ctx, at) - e.applyTimeouts(ctx, to) + e.applyTimeouts(ctx, at, to) } return nil } @@ -242,8 +242,8 @@ func (e *hcEventsObserver) applyWithConfidence(ctx context.Context, height abi.C } // Apply any timeouts that expire at this height -func (e *hcEventsObserver) applyTimeouts(ctx context.Context, ts *types.TipSet) { - triggers, ok := e.timeouts[ts.Height()] +func (e *hcEventsObserver) applyTimeouts(ctx context.Context, at abi.ChainEpoch, ts *types.TipSet) { + triggers, ok := e.timeouts[at] if !ok { return // nothing to do } @@ -258,14 +258,14 @@ func (e *hcEventsObserver) applyTimeouts(ctx context.Context, ts *types.TipSet) } // This should be cached. - timeoutTs, err := e.cs.ChainGetTipSetAfterHeight(ctx, ts.Height()-abi.ChainEpoch(trigger.confidence), ts.Key()) + timeoutTs, err := e.cs.ChainGetTipSetAfterHeight(ctx, at-abi.ChainEpoch(trigger.confidence), ts.Key()) if err != nil { - log.Errorf("events: applyTimeouts didn't find tipset for event; wanted %d; current %d", ts.Height()-abi.ChainEpoch(trigger.confidence), ts.Height()) + log.Errorf("events: applyTimeouts didn't find tipset for event; wanted %d; current %d", at-abi.ChainEpoch(trigger.confidence), at) } - more, err := trigger.handle(ctx, nil, nil, timeoutTs, ts.Height()) + more, err := trigger.handle(ctx, nil, nil, timeoutTs, at) if err != nil { - log.Errorf("chain trigger (call @H %d, called @ %d) failed: %s", timeoutTs.Height(), ts.Height(), err) + log.Errorf("chain trigger (call @H %d, called @ %d) failed: %s", timeoutTs.Height(), at, err) continue // don't revert failed calls } diff --git a/chain/events/events_test.go b/chain/events/events_test.go index 0536b5ebb..61dd25fbb 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -1255,72 +1255,80 @@ func TestStateChangedRevert(t *testing.T) { } func TestStateChangedTimeout(t *testing.T) { - fcs := newFakeCS(t) + timeoutHeight := abi.ChainEpoch(20) + confidence := 3 - events, err := NewEvents(context.Background(), fcs) - require.NoError(t, err) + testCases := []struct { + name string + checkFn CheckFunc + nilBlocks []int + expectTimeout bool + }{{ + // Verify that the state changed timeout is called at the expected height + name: "state changed timeout", + checkFn: func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { + return false, true, nil + }, + expectTimeout: true, + }, { + // Verify that the state changed timeout is called even if the timeout + // falls on nil block + name: "state changed timeout falls on nil block", + checkFn: func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { + return false, true, nil + }, + nilBlocks: []int{20, 21, 22, 23}, + expectTimeout: true, + }, { + // Verify that the state changed timeout is not called if the check + // function reports that it's complete + name: "no timeout callback if check func reports done", + checkFn: func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { + return true, true, nil + }, + expectTimeout: false, + }} - called := false + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + fcs := newFakeCS(t) - err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { - return false, true, nil - }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { - if data != nil { - require.Equal(t, oldTs.Key(), newTs.Parents()) - } - called = true - require.Nil(t, data) - require.Equal(t, abi.ChainEpoch(20), newTs.Height()) - require.Equal(t, abi.ChainEpoch(23), curH) - return false, nil - }, func(_ context.Context, ts *types.TipSet) error { - t.Fatal("revert on timeout") - return nil - }, 3, 20, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { - require.Equal(t, oldTs.Key(), newTs.Parents()) - return false, nil, nil - }) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) - require.NoError(t, err) + // Track whether the callback was called + called := false - fcs.advance(0, 21, 0, nil) - require.False(t, called) + // Set up state change tracking that will timeout at the given height + err = events.StateChanged( + tc.checkFn, + func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { + // Expect the callback to be called at the timeout height with nil data + called = true + require.Nil(t, data) + require.Equal(t, timeoutHeight, newTs.Height()) + require.Equal(t, timeoutHeight+abi.ChainEpoch(confidence), curH) + return false, nil + }, func(_ context.Context, ts *types.TipSet) error { + t.Fatal("revert on timeout") + return nil + }, confidence, timeoutHeight, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { + return false, nil, nil + }) - fcs.advance(0, 5, 0, nil) - require.True(t, called) - called = false + require.NoError(t, err) - // with check func reporting done + // Advance to timeout height + fcs.advance(0, int(timeoutHeight)+1, 0, nil) + require.False(t, called) - fcs = newFakeCS(t) - events, err = NewEvents(context.Background(), fcs) - require.NoError(t, err) - - err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { - return true, true, nil - }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { - if data != nil { - require.Equal(t, oldTs.Key(), newTs.Parents()) - } - called = true - require.Nil(t, data) - require.Equal(t, abi.ChainEpoch(20), newTs.Height()) - require.Equal(t, abi.ChainEpoch(23), curH) - return false, nil - }, func(_ context.Context, ts *types.TipSet) error { - t.Fatal("revert on timeout") - return nil - }, 3, 20, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { - require.Equal(t, oldTs.Key(), newTs.Parents()) - return false, nil, nil - }) - require.NoError(t, err) - - fcs.advance(0, 21, 0, nil) - require.False(t, called) - - fcs.advance(0, 5, 0, nil) - require.False(t, called) + // Advance past timeout height + fcs.advance(0, 5, 0, nil, tc.nilBlocks...) + require.Equal(t, tc.expectTimeout, called) + called = false + }) + } } func TestCalledMultiplePerEpoch(t *testing.T) {