diff --git a/api/api_gateway.go b/api/api_gateway.go index 29cd8ce24..862c6ddb5 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -33,6 +33,7 @@ type Gateway interface { ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) + ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*HeadChange, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetAfterHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) diff --git a/api/proxy_gen.go b/api/proxy_gen.go index f2dc7c560..c03b83531 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -480,6 +480,8 @@ type GatewayStruct struct { ChainGetMessage func(p0 context.Context, p1 cid.Cid) (*types.Message, error) `` + ChainGetPath func(p0 context.Context, p1 types.TipSetKey, p2 types.TipSetKey) ([]*HeadChange, error) `` + ChainGetTipSet func(p0 context.Context, p1 types.TipSetKey) (*types.TipSet, error) `` ChainGetTipSetAfterHeight func(p0 context.Context, p1 abi.ChainEpoch, p2 types.TipSetKey) (*types.TipSet, error) `` @@ -3039,6 +3041,17 @@ func (s *GatewayStub) ChainGetMessage(p0 context.Context, p1 cid.Cid) (*types.Me return nil, ErrNotSupported } +func (s *GatewayStruct) ChainGetPath(p0 context.Context, p1 types.TipSetKey, p2 types.TipSetKey) ([]*HeadChange, error) { + if s.Internal.ChainGetPath == nil { + return *new([]*HeadChange), ErrNotSupported + } + return s.Internal.ChainGetPath(p0, p1, p2) +} + +func (s *GatewayStub) ChainGetPath(p0 context.Context, p1 types.TipSetKey, p2 types.TipSetKey) ([]*HeadChange, error) { + return *new([]*HeadChange), ErrNotSupported +} + func (s *GatewayStruct) ChainGetTipSet(p0 context.Context, p1 types.TipSetKey) (*types.TipSet, error) { if s.Internal.ChainGetTipSet == nil { return nil, ErrNotSupported diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 4892e0901..610a0591a 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index b119be722..ce3a4562f 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 06d91c92a..84b1b1b1d 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/build/params_2k.go b/build/params_2k.go index efa38dc0c..479bd3b82 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -1,3 +1,4 @@ +//go:build debug || 2k // +build debug 2k package build diff --git a/build/params_butterfly.go b/build/params_butterfly.go index 4f4cc756d..1477c363a 100644 --- a/build/params_butterfly.go +++ b/build/params_butterfly.go @@ -1,3 +1,4 @@ +//go:build butterflynet // +build butterflynet package build diff --git a/build/params_calibnet.go b/build/params_calibnet.go index df334a516..bafed3015 100644 --- a/build/params_calibnet.go +++ b/build/params_calibnet.go @@ -1,3 +1,4 @@ +//go:build calibnet // +build calibnet package build diff --git a/build/params_debug.go b/build/params_debug.go index f679c9178..e977cda05 100644 --- a/build/params_debug.go +++ b/build/params_debug.go @@ -1,3 +1,4 @@ +//go:build debug // +build debug package build diff --git a/build/params_interop.go b/build/params_interop.go index 921dd0981..0040bb9d0 100644 --- a/build/params_interop.go +++ b/build/params_interop.go @@ -1,3 +1,4 @@ +//go:build interopnet // +build interopnet package build diff --git a/build/params_mainnet.go b/build/params_mainnet.go index 1c9b69462..1e9996082 100644 --- a/build/params_mainnet.go +++ b/build/params_mainnet.go @@ -1,10 +1,5 @@ -// +build !debug -// +build !2k -// +build !testground -// +build !calibnet -// +build !nerpanet -// +build !butterflynet -// +build !interopnet +//go:build !debug && !2k && !testground && !calibnet && !nerpanet && !butterflynet && !interopnet +// +build !debug,!2k,!testground,!calibnet,!nerpanet,!butterflynet,!interopnet package build diff --git a/build/params_nerpanet.go b/build/params_nerpanet.go index 5eef25cb2..b7beb5953 100644 --- a/build/params_nerpanet.go +++ b/build/params_nerpanet.go @@ -1,3 +1,4 @@ +//go:build nerpanet // +build nerpanet package build diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index db198e4aa..f3b6cb6a2 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -1,3 +1,4 @@ +//go:build !testground // +build !testground package build diff --git a/build/params_testground.go b/build/params_testground.go index 204c74e67..0bb984a27 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -1,3 +1,4 @@ +//go:build testground // +build testground // This file makes hardcoded parameters (const) configurable as vars. diff --git a/build/tools.go b/build/tools.go index 57b6e7d1f..adebe34dd 100644 --- a/build/tools.go +++ b/build/tools.go @@ -1,4 +1,5 @@ -//+build tools +//go:build tools +// +build tools package build diff --git a/chain/events/cache.go b/chain/events/cache.go new file mode 100644 index 000000000..ef4b5bba8 --- /dev/null +++ b/chain/events/cache.go @@ -0,0 +1,33 @@ +package events + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" +) + +type uncachedAPI interface { + ChainNotify(context.Context) (<-chan []*api.HeadChange, error) + ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) + StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) + + StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // optional / for CalledMsg +} + +type cache struct { + *tipSetCache + *messageCache + uncachedAPI +} + +func newCache(api EventAPI, gcConfidence abi.ChainEpoch) *cache { + return &cache{ + newTSCache(api, gcConfidence), + newMessageCache(api), + api, + } +} diff --git a/chain/events/events.go b/chain/events/events.go index 8511de921..5c494fcb0 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -2,18 +2,14 @@ package events import ( "context" - "sync" - "time" "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) @@ -25,209 +21,46 @@ type ( RevertHandler func(ctx context.Context, ts *types.TipSet) error ) -type heightHandler struct { - confidence int - called bool - - handle HeightHandler - revert RevertHandler +// A TipSetObserver receives notifications of tipsets +type TipSetObserver interface { + Apply(ctx context.Context, from, to *types.TipSet) error + Revert(ctx context.Context, from, to *types.TipSet) error } type EventAPI interface { ChainNotify(context.Context) (<-chan []*api.HeadChange, error) ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) ChainHead(context.Context) (*types.TipSet, error) StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error) + ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // optional / for CalledMsg } type Events struct { - api EventAPI - - tsc *tipSetCache - lk sync.Mutex - - ready chan struct{} - readyOnce sync.Once - - heightEvents + *observer + *heightEvents *hcEvents - - observers []TipSetObserver } -func NewEventsWithConfidence(ctx context.Context, api EventAPI, gcConfidence abi.ChainEpoch) *Events { - tsc := newTSCache(gcConfidence, api) +func NewEventsWithConfidence(ctx context.Context, api EventAPI, gcConfidence abi.ChainEpoch) (*Events, error) { + cache := newCache(api, gcConfidence) - e := &Events{ - api: api, - - tsc: tsc, - - heightEvents: heightEvents{ - tsc: tsc, - ctx: ctx, - gcConfidence: gcConfidence, - - heightTriggers: map[uint64]*heightHandler{}, - htTriggerHeights: map[abi.ChainEpoch][]uint64{}, - htHeights: map[abi.ChainEpoch][]uint64{}, - }, - - hcEvents: newHCEvents(ctx, api, tsc, uint64(gcConfidence)), - ready: make(chan struct{}), - observers: []TipSetObserver{}, + ob := newObserver(cache, gcConfidence) + if err := ob.start(ctx); err != nil { + return nil, err } - go e.listenHeadChanges(ctx) + he := newHeightEvents(cache, ob, gcConfidence) + headChange := newHCEvents(cache, ob) - // Wait for the first tipset to be seen or bail if shutting down - select { - case <-e.ready: - case <-ctx.Done(): - } - - return e + return &Events{ob, he, headChange}, nil } -func NewEvents(ctx context.Context, api EventAPI) *Events { +func NewEvents(ctx context.Context, api EventAPI) (*Events, error) { gcConfidence := 2 * build.ForkLengthThreshold return NewEventsWithConfidence(ctx, api, gcConfidence) } - -func (e *Events) listenHeadChanges(ctx context.Context) { - for { - if err := e.listenHeadChangesOnce(ctx); err != nil { - log.Errorf("listen head changes errored: %s", err) - } else { - log.Warn("listenHeadChanges quit") - } - select { - case <-build.Clock.After(time.Second): - case <-ctx.Done(): - log.Warnf("not restarting listenHeadChanges: context error: %s", ctx.Err()) - return - } - - log.Info("restarting listenHeadChanges") - } -} - -func (e *Events) listenHeadChangesOnce(ctx context.Context) error { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - notifs, err := e.api.ChainNotify(ctx) - if err != nil { - // Retry is handled by caller - return xerrors.Errorf("listenHeadChanges ChainNotify call failed: %w", err) - } - - var cur []*api.HeadChange - var ok bool - - // Wait for first tipset or bail - select { - case cur, ok = <-notifs: - if !ok { - return xerrors.Errorf("notification channel closed") - } - case <-ctx.Done(): - return ctx.Err() - } - - if len(cur) != 1 { - return xerrors.Errorf("unexpected initial head notification length: %d", len(cur)) - } - - if cur[0].Type != store.HCCurrent { - return xerrors.Errorf("expected first head notification type to be 'current', was '%s'", cur[0].Type) - } - - if err := e.tsc.add(cur[0].Val); err != nil { - log.Warnf("tsc.add: adding current tipset failed: %v", err) - } - - e.readyOnce.Do(func() { - e.lastTs = cur[0].Val - // Signal that we have seen first tipset - close(e.ready) - }) - - for notif := range notifs { - var rev, app []*types.TipSet - for _, notif := range notif { - switch notif.Type { - case store.HCRevert: - rev = append(rev, notif.Val) - case store.HCApply: - app = append(app, notif.Val) - default: - log.Warnf("unexpected head change notification type: '%s'", notif.Type) - } - } - - if err := e.headChange(ctx, rev, app); err != nil { - log.Warnf("headChange failed: %s", err) - } - - // sync with fake chainstore (for tests) - if fcs, ok := e.api.(interface{ notifDone() }); ok { - fcs.notifDone() - } - } - - return nil -} - -func (e *Events) headChange(ctx context.Context, rev, app []*types.TipSet) error { - if len(app) == 0 { - return xerrors.New("events.headChange expected at least one applied tipset") - } - - e.lk.Lock() - defer e.lk.Unlock() - - if err := e.headChangeAt(rev, app); err != nil { - return err - } - - if err := e.observeChanges(ctx, rev, app); err != nil { - return err - } - return e.processHeadChangeEvent(rev, app) -} - -// A TipSetObserver receives notifications of tipsets -type TipSetObserver interface { - Apply(ctx context.Context, ts *types.TipSet) error - Revert(ctx context.Context, ts *types.TipSet) error -} - -// TODO: add a confidence level so we can have observers with difference levels of confidence -func (e *Events) Observe(obs TipSetObserver) error { - e.lk.Lock() - defer e.lk.Unlock() - e.observers = append(e.observers, obs) - return nil -} - -// observeChanges expects caller to hold e.lk -func (e *Events) observeChanges(ctx context.Context, rev, app []*types.TipSet) error { - for _, ts := range rev { - for _, o := range e.observers { - _ = o.Revert(ctx, ts) - } - } - - for _, ts := range app { - for _, o := range e.observers { - _ = o.Apply(ctx, ts) - } - } - - return nil -} diff --git a/chain/events/events_called.go b/chain/events/events_called.go index 1f0b80169..ffca57d5b 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -5,9 +5,6 @@ import ( "math" "sync" - "github.com/filecoin-project/lotus/api" - lru "github.com/hashicorp/golang-lru" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/go-state-types/abi" @@ -35,7 +32,7 @@ type eventData interface{} // `prevTs` is the previous tipset, eg the "from" tipset for a state change. // `ts` is the event tipset, eg the tipset in which the `msg` is included. // `curH`-`ts.Height` = `confidence` -type EventHandler func(data eventData, prevTs, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error) +type EventHandler func(ctx context.Context, data eventData, prevTs, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error) // CheckFunc is used for atomicity guarantees. If the condition the callbacks // wait for has already happened in tipset `ts` @@ -43,7 +40,7 @@ type EventHandler func(data eventData, prevTs, ts *types.TipSet, curH abi.ChainE // If `done` is true, timeout won't be triggered // If `more` is false, no messages will be sent to EventHandler (RevertHandler // may still be called) -type CheckFunc func(ts *types.TipSet) (done bool, more bool, err error) +type CheckFunc func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) // Keep track of information for an event handler type handlerInfo struct { @@ -60,10 +57,9 @@ type handlerInfo struct { // until the required confidence is reached type queuedEvent struct { trigger triggerID + data eventData - prevH abi.ChainEpoch - h abi.ChainEpoch - data eventData + prevTipset, tipset *types.TipSet called bool } @@ -71,19 +67,17 @@ type queuedEvent struct { // Manages chain head change events, which may be forward (new tipset added to // chain) or backward (chain branch discarded in favour of heavier branch) type hcEvents struct { - cs EventAPI - tsc *tipSetCache - ctx context.Context - gcConfidence uint64 + cs EventAPI + lk sync.Mutex lastTs *types.TipSet - lk sync.Mutex - ctr triggerID + // TODO: get rid of trigger IDs and just use pointers as keys. triggers map[triggerID]*handlerInfo + // TODO: instead of scheduling events in the future, look at the chain in the past. We can sip the "confidence" queue entirely. // maps block heights to events // [triggerH][msgH][event] confQueue map[triggerH]map[msgH][]*queuedEvent @@ -98,83 +92,77 @@ type hcEvents struct { watcherEvents } -func newHCEvents(ctx context.Context, cs EventAPI, tsc *tipSetCache, gcConfidence uint64) *hcEvents { - e := hcEvents{ - ctx: ctx, - cs: cs, - tsc: tsc, - gcConfidence: gcConfidence, - +func newHCEvents(api EventAPI, obs *observer) *hcEvents { + e := &hcEvents{ + cs: api, confQueue: map[triggerH]map[msgH][]*queuedEvent{}, revertQueue: map[msgH][]triggerH{}, triggers: map[triggerID]*handlerInfo{}, timeouts: map[abi.ChainEpoch]map[triggerID]int{}, } - e.messageEvents = newMessageEvents(ctx, &e, cs) - e.watcherEvents = newWatcherEvents(ctx, &e, cs) + e.messageEvents = newMessageEvents(e, api) + e.watcherEvents = newWatcherEvents(e, api) - return &e + // We need to take the lock as the observer could immediately try calling us. + e.lk.Lock() + e.lastTs = obs.Observe((*hcEventsObserver)(e)) + e.lk.Unlock() + + return e } -// Called when there is a change to the head with tipsets to be -// reverted / applied -func (e *hcEvents) processHeadChangeEvent(rev, app []*types.TipSet) error { +type hcEventsObserver hcEvents + +func (e *hcEventsObserver) Apply(ctx context.Context, from, to *types.TipSet) error { e.lk.Lock() defer e.lk.Unlock() - for _, ts := range rev { - e.handleReverts(ts) - e.lastTs = ts + defer func() { e.lastTs = to }() + + // Check if the head change caused any state changes that we were + // waiting for + stateChanges := e.checkStateChanges(from, to) + + // Queue up calls until there have been enough blocks to reach + // confidence on the state changes + for tid, data := range stateChanges { + e.queueForConfidence(tid, data, from, to) } - for _, ts := range app { - // Check if the head change caused any state changes that we were - // waiting for - stateChanges := e.watcherEvents.checkStateChanges(e.lastTs, ts) + // Check if the head change included any new message calls + newCalls := e.checkNewCalls(ctx, from, to) - // Queue up calls until there have been enough blocks to reach - // confidence on the state changes - for tid, data := range stateChanges { - e.queueForConfidence(tid, data, e.lastTs, ts) + // Queue up calls until there have been enough blocks to reach + // confidence on the message calls + for tid, calls := range newCalls { + for _, data := range calls { + e.queueForConfidence(tid, data, nil, to) } - - // Check if the head change included any new message calls - newCalls, err := e.messageEvents.checkNewCalls(ts) - if err != nil { - return err - } - - // Queue up calls until there have been enough blocks to reach - // confidence on the message calls - for tid, calls := range newCalls { - for _, data := range calls { - e.queueForConfidence(tid, data, nil, ts) - } - } - - for at := e.lastTs.Height(); at <= ts.Height(); at++ { - // Apply any queued events and timeouts that were targeted at the - // current chain height - e.applyWithConfidence(ts, at) - e.applyTimeouts(ts) - } - - // Update the latest known tipset - e.lastTs = ts } + for at := from.Height() + 1; at <= to.Height(); at++ { + // Apply any queued events and timeouts that were targeted at the + // current chain height + e.applyWithConfidence(ctx, at) + e.applyTimeouts(ctx, at, to) + } return nil } -func (e *hcEvents) handleReverts(ts *types.TipSet) { - reverts, ok := e.revertQueue[ts.Height()] +func (e *hcEventsObserver) Revert(ctx context.Context, from, to *types.TipSet) error { + e.lk.Lock() + defer e.lk.Unlock() + + defer func() { e.lastTs = to }() + + reverts, ok := e.revertQueue[from.Height()] if !ok { - return // nothing to do + return nil // nothing to do } for _, triggerH := range reverts { - toRevert := e.confQueue[triggerH][ts.Height()] + toRevert := e.confQueue[triggerH][from.Height()] for _, event := range toRevert { if !event.called { continue // event wasn't apply()-ied yet @@ -182,24 +170,21 @@ func (e *hcEvents) handleReverts(ts *types.TipSet) { trigger := e.triggers[event.trigger] - if err := trigger.revert(e.ctx, ts); err != nil { - log.Errorf("reverting chain trigger (@H %d, triggered @ %d) failed: %s", ts.Height(), triggerH, err) + if err := trigger.revert(ctx, from); err != nil { + log.Errorf("reverting chain trigger (@H %d, triggered @ %d) failed: %s", from.Height(), triggerH, err) } } - delete(e.confQueue[triggerH], ts.Height()) + delete(e.confQueue[triggerH], from.Height()) } - delete(e.revertQueue, ts.Height()) + delete(e.revertQueue, from.Height()) + return nil } // Queue up events until the chain has reached a height that reflects the // desired confidence -func (e *hcEvents) queueForConfidence(trigID uint64, data eventData, prevTs, ts *types.TipSet) { +func (e *hcEventsObserver) queueForConfidence(trigID uint64, data eventData, prevTs, ts *types.TipSet) { trigger := e.triggers[trigID] - prevH := NoHeight - if prevTs != nil { - prevH = prevTs.Height() - } appliedH := ts.Height() triggerH := appliedH + abi.ChainEpoch(trigger.confidence) @@ -211,28 +196,23 @@ func (e *hcEvents) queueForConfidence(trigID uint64, data eventData, prevTs, ts } byOrigH[appliedH] = append(byOrigH[appliedH], &queuedEvent{ - trigger: trigID, - prevH: prevH, - h: appliedH, - data: data, + trigger: trigID, + data: data, + tipset: ts, + prevTipset: prevTs, }) e.revertQueue[appliedH] = append(e.revertQueue[appliedH], triggerH) } // Apply any events that were waiting for this chain height for confidence -func (e *hcEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpoch) { +func (e *hcEventsObserver) applyWithConfidence(ctx context.Context, height abi.ChainEpoch) { byOrigH, ok := e.confQueue[height] if !ok { return // no triggers at this height } for origH, events := range byOrigH { - triggerTs, err := e.tsc.get(origH) - if err != nil { - log.Errorf("events: applyWithConfidence didn't find tipset for event; wanted %d; current %d", origH, height) - } - for _, event := range events { if event.called { continue @@ -243,18 +223,7 @@ func (e *hcEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpoch) continue } - // Previous tipset - this is relevant for example in a state change - // from one tipset to another - var prevTs *types.TipSet - if event.prevH != NoHeight { - prevTs, err = e.tsc.get(event.prevH) - if err != nil { - log.Errorf("events: applyWithConfidence didn't find tipset for previous event; wanted %d; current %d", event.prevH, height) - continue - } - } - - more, err := trigger.handle(event.data, prevTs, triggerTs, height) + more, err := trigger.handle(ctx, event.data, event.prevTipset, event.tipset, height) if err != nil { log.Errorf("chain trigger (@H %d, triggered @ %d) failed: %s", origH, height, err) continue // don't revert failed calls @@ -273,8 +242,8 @@ func (e *hcEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpoch) } // Apply any timeouts that expire at this height -func (e *hcEvents) applyTimeouts(ts *types.TipSet) { - triggers, ok := e.timeouts[ts.Height()] +func (e *hcEventsObserver) applyTimeouts(ctx context.Context, at abi.ChainEpoch, ts *types.TipSet) { + triggers, ok := e.timeouts[at] if !ok { return // nothing to do } @@ -288,14 +257,15 @@ func (e *hcEvents) applyTimeouts(ts *types.TipSet) { continue } - timeoutTs, err := e.tsc.get(ts.Height() - abi.ChainEpoch(trigger.confidence)) + // This should be cached. + timeoutTs, err := e.cs.ChainGetTipSetAfterHeight(ctx, at-abi.ChainEpoch(trigger.confidence), ts.Key()) if err != nil { - log.Errorf("events: applyTimeouts didn't find tipset for event; wanted %d; current %d", ts.Height()-abi.ChainEpoch(trigger.confidence), ts.Height()) + log.Errorf("events: applyTimeouts didn't find tipset for event; wanted %d; current %d", at-abi.ChainEpoch(trigger.confidence), at) } - more, err := trigger.handle(nil, nil, timeoutTs, ts.Height()) + more, err := trigger.handle(ctx, nil, nil, timeoutTs, at) if err != nil { - log.Errorf("chain trigger (call @H %d, called @ %d) failed: %s", timeoutTs.Height(), ts.Height(), err) + log.Errorf("chain trigger (call @H %d, called @ %d) failed: %s", timeoutTs.Height(), at, err) continue // don't revert failed calls } @@ -309,24 +279,19 @@ func (e *hcEvents) applyTimeouts(ts *types.TipSet) { // - RevertHandler: called if the chain head changes causing the event to revert // - confidence: wait this many tipsets before calling EventHandler // - timeout: at this chain height, timeout on waiting for this event -func (e *hcEvents) onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) { +func (e *hcEvents) onHeadChanged(ctx context.Context, check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) { e.lk.Lock() defer e.lk.Unlock() // Check if the event has already occurred - ts, err := e.tsc.best() + done, more, err := check(ctx, e.lastTs) if err != nil { - return 0, xerrors.Errorf("error getting best tipset: %w", err) - } - done, more, err := check(ts) - if err != nil { - return 0, xerrors.Errorf("called check error (h: %d): %w", ts.Height(), err) + return 0, xerrors.Errorf("called check error (h: %d): %w", e.lastTs.Height(), err) } if done { timeout = NoTimeout } - // Create a trigger for the event id := e.ctr e.ctr++ @@ -354,12 +319,11 @@ func (e *hcEvents) onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHa // headChangeAPI is used to allow the composed event APIs to call back to hcEvents // to listen for changes type headChangeAPI interface { - onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) + onHeadChanged(ctx context.Context, check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) } // watcherEvents watches for a state change type watcherEvents struct { - ctx context.Context cs EventAPI hcAPI headChangeAPI @@ -367,9 +331,8 @@ type watcherEvents struct { matchers map[triggerID]StateMatchFunc } -func newWatcherEvents(ctx context.Context, hcAPI headChangeAPI, cs EventAPI) watcherEvents { +func newWatcherEvents(hcAPI headChangeAPI, cs EventAPI) watcherEvents { return watcherEvents{ - ctx: ctx, cs: cs, hcAPI: hcAPI, matchers: make(map[triggerID]StateMatchFunc), @@ -425,7 +388,7 @@ type StateMatchFunc func(oldTs, newTs *types.TipSet) (bool, StateChange, error) // * `StateChangeHandler` is called when the specified state change was observed // on-chain, and a confidence threshold was reached, or the specified `timeout` // height was reached with no state change observed. When this callback is -// invoked on a timeout, `oldState` and `newState` are set to nil. +// invoked on a timeout, `oldTs` and `states are set to nil. // This callback returns a boolean specifying whether further notifications // should be sent, like `more` return param from `CheckFunc` above. // @@ -438,7 +401,7 @@ type StateMatchFunc func(oldTs, newTs *types.TipSet) (bool, StateChange, error) // the state change is queued up until the confidence interval has elapsed (and // `StateChangeHandler` is called) func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf StateMatchFunc) error { - hnd := func(data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) { + hnd := func(ctx context.Context, data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) { states, ok := data.(StateChange) if data != nil && !ok { panic("expected StateChange") @@ -447,7 +410,7 @@ func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler, return scHnd(prevTs, ts, states, height) } - id, err := we.hcAPI.onHeadChanged(check, hnd, rev, confidence, timeout) + id, err := we.hcAPI.onHeadChanged(context.TODO(), check, hnd, rev, confidence, timeout) if err != nil { return err } @@ -461,43 +424,29 @@ func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler, // messageEvents watches for message calls to actors type messageEvents struct { - ctx context.Context cs EventAPI hcAPI headChangeAPI lk sync.RWMutex matchers map[triggerID]MsgMatchFunc - - blockMsgLk sync.Mutex - blockMsgCache *lru.ARCCache } -func newMessageEvents(ctx context.Context, hcAPI headChangeAPI, cs EventAPI) messageEvents { - blsMsgCache, _ := lru.NewARC(500) +func newMessageEvents(hcAPI headChangeAPI, cs EventAPI) messageEvents { return messageEvents{ - ctx: ctx, - cs: cs, - hcAPI: hcAPI, - matchers: make(map[triggerID]MsgMatchFunc), - blockMsgLk: sync.Mutex{}, - blockMsgCache: blsMsgCache, + cs: cs, + hcAPI: hcAPI, + matchers: make(map[triggerID]MsgMatchFunc), } } // Check if there are any new actor calls -func (me *messageEvents) checkNewCalls(ts *types.TipSet) (map[triggerID][]eventData, error) { - pts, err := me.cs.ChainGetTipSet(me.ctx, ts.Parents()) // we actually care about messages in the parent tipset here - if err != nil { - log.Errorf("getting parent tipset in checkNewCalls: %s", err) - return nil, err - } - +func (me *messageEvents) checkNewCalls(ctx context.Context, from, to *types.TipSet) map[triggerID][]eventData { me.lk.RLock() defer me.lk.RUnlock() // For each message in the tipset res := make(map[triggerID][]eventData) - me.messagesForTs(pts, func(msg *types.Message) { + me.messagesForTs(from, func(msg *types.Message) { // TODO: provide receipts // Run each trigger's matcher against the message @@ -516,47 +465,32 @@ func (me *messageEvents) checkNewCalls(ts *types.TipSet) (map[triggerID][]eventD } }) - return res, nil + return res } // Get the messages in a tipset func (me *messageEvents) messagesForTs(ts *types.TipSet, consume func(*types.Message)) { seen := map[cid.Cid]struct{}{} - for _, tsb := range ts.Blocks() { - me.blockMsgLk.Lock() - msgsI, ok := me.blockMsgCache.Get(tsb.Cid()) - var err error - if !ok { - msgsI, err = me.cs.ChainGetBlockMessages(context.TODO(), tsb.Cid()) - if err != nil { - log.Errorf("messagesForTs MessagesForBlock failed (ts.H=%d, Bcid:%s, B.Mcid:%s): %s", ts.Height(), tsb.Cid(), tsb.Messages, err) - // this is quite bad, but probably better than missing all the other updates - me.blockMsgLk.Unlock() - continue - } - me.blockMsgCache.Add(tsb.Cid(), msgsI) + for i, tsb := range ts.Cids() { + msgs, err := me.cs.ChainGetBlockMessages(context.TODO(), tsb) + if err != nil { + log.Errorf("messagesForTs MessagesForBlock failed (ts.H=%d, Bcid:%s, B.Mcid:%s): %s", + ts.Height(), tsb, ts.Blocks()[i].Messages, err) + continue } - me.blockMsgLk.Unlock() - msgs := msgsI.(*api.BlockMessages) - for _, m := range msgs.BlsMessages { - _, ok := seen[m.Cid()] + for i, c := range msgs.Cids { + // We iterate over the CIDs to avoid having to recompute them. + _, ok := seen[c] if ok { continue } - seen[m.Cid()] = struct{}{} - - consume(m) - } - - for _, m := range msgs.SecpkMessages { - _, ok := seen[m.Message.Cid()] - if ok { - continue + seen[c] = struct{}{} + if i < len(msgs.BlsMessages) { + consume(msgs.BlsMessages[i]) + } else { + consume(&msgs.SecpkMessages[i-len(msgs.BlsMessages)].Message) } - seen[m.Message.Cid()] = struct{}{} - - consume(&m.Message) } } } @@ -596,14 +530,14 @@ type MsgMatchFunc func(msg *types.Message) (matched bool, err error) // * `MsgMatchFunc` is called against each message. If there is a match, the // message is queued up until the confidence interval has elapsed (and // `MsgHandler` is called) -func (me *messageEvents) Called(check CheckFunc, msgHnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf MsgMatchFunc) error { - hnd := func(data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) { +func (me *messageEvents) Called(ctx context.Context, check CheckFunc, msgHnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf MsgMatchFunc) error { + hnd := func(ctx context.Context, data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) { msg, ok := data.(*types.Message) if data != nil && !ok { panic("expected msg") } - ml, err := me.cs.StateSearchMsg(me.ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true) + ml, err := me.cs.StateSearchMsg(ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true) if err != nil { return false, err } @@ -615,7 +549,7 @@ func (me *messageEvents) Called(check CheckFunc, msgHnd MsgHandler, rev RevertHa return msgHnd(msg, &ml.Receipt, ts, height) } - id, err := me.hcAPI.onHeadChanged(check, hnd, rev, confidence, timeout) + id, err := me.hcAPI.onHeadChanged(ctx, check, hnd, rev, confidence, timeout) if err != nil { return err } @@ -629,5 +563,5 @@ func (me *messageEvents) Called(check CheckFunc, msgHnd MsgHandler, rev RevertHa // Convenience function for checking and matching messages func (me *messageEvents) CalledMsg(ctx context.Context, hnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, msg types.ChainMsg) error { - return me.Called(me.CheckMsg(ctx, msg, hnd), hnd, rev, confidence, timeout, me.MatchMsg(msg.VMMessage())) + return me.Called(ctx, me.CheckMsg(msg, hnd), hnd, rev, confidence, timeout, me.MatchMsg(msg.VMMessage())) } diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 1fcff9e68..6734d3ca0 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -11,199 +11,235 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -type heightEvents struct { - lk sync.Mutex - tsc *tipSetCache - gcConfidence abi.ChainEpoch +type heightHandler struct { + ts *types.TipSet + height abi.ChainEpoch + called bool - ctr triggerID - - heightTriggers map[triggerID]*heightHandler - - htTriggerHeights map[triggerH][]triggerID - htHeights map[msgH][]triggerID - - ctx context.Context + handle HeightHandler + revert RevertHandler } -func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { - ctx, span := trace.StartSpan(e.ctx, "events.HeightHeadChange") - defer span.End() - span.AddAttributes(trace.Int64Attribute("endHeight", int64(app[0].Height()))) - span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev)))) - span.AddAttributes(trace.Int64Attribute("applies", int64(len(app)))) +type heightEvents struct { + api EventAPI + gcConfidence abi.ChainEpoch - e.lk.Lock() - defer e.lk.Unlock() - for _, ts := range rev { - // TODO: log error if h below gcconfidence - // revert height-based triggers + lk sync.Mutex + head *types.TipSet + tsHeights, triggerHeights map[abi.ChainEpoch][]*heightHandler + lastGc abi.ChainEpoch //nolint:structcheck +} - revert := func(h abi.ChainEpoch, ts *types.TipSet) { - for _, tid := range e.htHeights[h] { - ctx, span := trace.StartSpan(ctx, "events.HeightRevert") - - rev := e.heightTriggers[tid].revert - e.lk.Unlock() - err := rev(ctx, ts) - e.lk.Lock() - e.heightTriggers[tid].called = false - - span.End() - - if err != nil { - log.Errorf("reverting chain trigger (@H %d): %s", h, err) - } - } - } - revert(ts.Height(), ts) - - subh := ts.Height() - 1 - for { - cts, err := e.tsc.get(subh) - if err != nil { - return err - } - - if cts != nil { - break - } - - revert(subh, ts) - subh-- - } - - if err := e.tsc.revert(ts); err != nil { - return err - } +func newHeightEvents(api EventAPI, obs *observer, gcConfidence abi.ChainEpoch) *heightEvents { + he := &heightEvents{ + api: api, + gcConfidence: gcConfidence, + tsHeights: map[abi.ChainEpoch][]*heightHandler{}, + triggerHeights: map[abi.ChainEpoch][]*heightHandler{}, } - - for i := range app { - ts := app[i] - - if err := e.tsc.add(ts); err != nil { - return err - } - - // height triggers - - apply := func(h abi.ChainEpoch, ts *types.TipSet) error { - for _, tid := range e.htTriggerHeights[h] { - hnd := e.heightTriggers[tid] - if hnd.called { - return nil - } - - triggerH := h - abi.ChainEpoch(hnd.confidence) - - incTs, err := e.tsc.getNonNull(triggerH) - if err != nil { - return err - } - - ctx, span := trace.StartSpan(ctx, "events.HeightApply") - span.AddAttributes(trace.BoolAttribute("immediate", false)) - handle := hnd.handle - e.lk.Unlock() - err = handle(ctx, incTs, h) - e.lk.Lock() - hnd.called = true - span.End() - - if err != nil { - log.Errorf("chain trigger (@H %d, called @ %d) failed: %+v", triggerH, ts.Height(), err) - } - } - return nil - } - - if err := apply(ts.Height(), ts); err != nil { - return err - } - subh := ts.Height() - 1 - for { - cts, err := e.tsc.get(subh) - if err != nil { - return err - } - - if cts != nil { - break - } - - if err := apply(subh, ts); err != nil { - return err - } - - subh-- - } - - } - - return nil + he.lk.Lock() + he.head = obs.Observe((*heightEventsObserver)(he)) + he.lk.Unlock() + return he } // ChainAt invokes the specified `HeightHandler` when the chain reaches the // specified height+confidence threshold. If the chain is rolled-back under the // specified height, `RevertHandler` will be called. // -// ts passed to handlers is the tipset at the specified, or above, if lower tipsets were null -func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence int, h abi.ChainEpoch) error { - e.lk.Lock() // Tricky locking, check your locks if you modify this function! - - best, err := e.tsc.best() - if err != nil { - e.lk.Unlock() - return xerrors.Errorf("error getting best tipset: %w", err) +// ts passed to handlers is the tipset at the specified epoch, or above if lower tipsets were null. +// +// The context governs cancellations of this call, it won't cancel the event handler. +func (e *heightEvents) ChainAt(ctx context.Context, hnd HeightHandler, rev RevertHandler, confidence int, h abi.ChainEpoch) error { + if abi.ChainEpoch(confidence) > e.gcConfidence { + // Need this to be able to GC effectively. + return xerrors.Errorf("confidence cannot be greater than gcConfidence: %d > %d", confidence, e.gcConfidence) } - - bestH := best.Height() - if bestH >= h+abi.ChainEpoch(confidence) { - ts, err := e.tsc.getNonNull(h) - if err != nil { - log.Warnf("events.ChainAt: calling HandleFunc with nil tipset, not found in cache: %s", err) - } - - e.lk.Unlock() - ctx, span := trace.StartSpan(e.ctx, "events.HeightApply") - span.AddAttributes(trace.BoolAttribute("immediate", true)) - - err = hnd(ctx, ts, bestH) - span.End() - - if err != nil { - return err - } - - e.lk.Lock() - best, err = e.tsc.best() - if err != nil { - e.lk.Unlock() - return xerrors.Errorf("error getting best tipset: %w", err) - } - bestH = best.Height() - } - - defer e.lk.Unlock() - - if bestH >= h+abi.ChainEpoch(confidence)+e.gcConfidence { - return nil - } - - triggerAt := h + abi.ChainEpoch(confidence) - - id := e.ctr - e.ctr++ - - e.heightTriggers[id] = &heightHandler{ - confidence: confidence, - + handler := &heightHandler{ + height: h, handle: hnd, revert: rev, } + triggerAt := h + abi.ChainEpoch(confidence) - e.htHeights[h] = append(e.htHeights[h], id) - e.htTriggerHeights[triggerAt] = append(e.htTriggerHeights[triggerAt], id) + // Here we try to jump onto a moving train. To avoid stopping the train, we release the lock + // while calling the API and/or the trigger functions. Unfortunately, it's entirely possible + // (although unlikely) to go back and forth across the trigger heights, so we need to keep + // going back and forth here till we're synced. + // + // TODO: Consider using a worker goroutine so we can just drop the handler in a channel? The + // downside is that we'd either need a tipset cache, or we'd need to potentially fetch + // tipsets in-line inside the event loop. + e.lk.Lock() + for { + head := e.head + if head.Height() >= h { + // Head is past the handler height. We at least need to stash the tipset to + // avoid doing this from the main event loop. + e.lk.Unlock() + var ts *types.TipSet + if head.Height() == h { + ts = head + } else { + var err error + ts, err = e.api.ChainGetTipSetAfterHeight(ctx, handler.height, head.Key()) + if err != nil { + return xerrors.Errorf("events.ChainAt: failed to get tipset: %s", err) + } + } + + // If we've applied the handler on the wrong tipset, revert. + if handler.called && !ts.Equals(handler.ts) { + ctx, span := trace.StartSpan(ctx, "events.HeightRevert") + span.AddAttributes(trace.BoolAttribute("immediate", true)) + err := handler.revert(ctx, handler.ts) + span.End() + if err != nil { + return err + } + handler.called = false + } + + // Save the tipset. + handler.ts = ts + + // If we've reached confidence and haven't called, call. + if !handler.called && head.Height() >= triggerAt { + ctx, span := trace.StartSpan(ctx, "events.HeightApply") + span.AddAttributes(trace.BoolAttribute("immediate", true)) + err := handler.handle(ctx, handler.ts, head.Height()) + span.End() + if err != nil { + return err + } + + handler.called = true + + // If we've reached gcConfidence, return without saving anything. + if head.Height() >= h+e.gcConfidence { + return nil + } + } + + e.lk.Lock() + } else if handler.called { + // We're not passed the head (anymore) but have applied the handler. Revert, try again. + e.lk.Unlock() + ctx, span := trace.StartSpan(ctx, "events.HeightRevert") + span.AddAttributes(trace.BoolAttribute("immediate", true)) + err := handler.revert(ctx, handler.ts) + span.End() + if err != nil { + return err + } + handler.called = false + e.lk.Lock() + } // otherwise, we changed heads but the change didn't matter. + + // If we managed to get through this without the head changing, we're finally done. + if head.Equals(e.head) { + e.triggerHeights[triggerAt] = append(e.triggerHeights[triggerAt], handler) + e.tsHeights[h] = append(e.tsHeights[h], handler) + e.lk.Unlock() + return nil + } + } +} + +// Updates the head and garbage collects if we're 2x over our garbage collection confidence period. +func (e *heightEventsObserver) updateHead(h *types.TipSet) { + e.lk.Lock() + defer e.lk.Unlock() + e.head = h + + if e.head.Height() < e.lastGc+e.gcConfidence*2 { + return + } + e.lastGc = h.Height() + + targetGcHeight := e.head.Height() - e.gcConfidence + for h := range e.tsHeights { + if h >= targetGcHeight { + continue + } + delete(e.tsHeights, h) + } + for h := range e.triggerHeights { + if h >= targetGcHeight { + continue + } + delete(e.triggerHeights, h) + } +} + +type heightEventsObserver heightEvents + +func (e *heightEventsObserver) Revert(ctx context.Context, from, to *types.TipSet) error { + // Update the head first so we don't accidental skip reverting a concurrent call to ChainAt. + e.updateHead(to) + + // Call revert on all hights between the two tipsets, handling empty tipsets. + for h := from.Height(); h > to.Height(); h-- { + e.lk.Lock() + triggers := e.tsHeights[h] + e.lk.Unlock() + + // 1. Triggers are only invoked from the global event loop, we don't need to hold the lock while calling. + // 2. We only ever append to or replace the trigger slice, so it's safe to iterate over it without the lock. + for _, handler := range triggers { + handler.ts = nil // invalidate + if !handler.called { + // We haven't triggered this yet, or there has been a concurrent call to ChainAt. + continue + } + ctx, span := trace.StartSpan(ctx, "events.HeightRevert") + err := handler.revert(ctx, from) + span.End() + + if err != nil { + log.Errorf("reverting chain trigger (@H %d): %s", h, err) + } + handler.called = false + } + } + return nil +} + +func (e *heightEventsObserver) Apply(ctx context.Context, from, to *types.TipSet) error { + // Update the head first so we don't accidental skip applying a concurrent call to ChainAt. + e.updateHead(to) + + for h := from.Height() + 1; h <= to.Height(); h++ { + e.lk.Lock() + triggers := e.triggerHeights[h] + tipsets := e.tsHeights[h] + e.lk.Unlock() + + // Stash the tipset for future triggers. + for _, handler := range tipsets { + handler.ts = to + } + + // Trigger the ready triggers. + for _, handler := range triggers { + if handler.called { + // We may have reverted past the trigger point, but not past the call point. + // Or there has been a concurrent call to ChainAt. + continue + } + + ctx, span := trace.StartSpan(ctx, "events.HeightApply") + span.AddAttributes(trace.BoolAttribute("immediate", false)) + err := handler.handle(ctx, handler.ts, h) + span.End() + + if err != nil { + log.Errorf("chain trigger (@H %d, called @ %d) failed: %+v", h, to.Height(), err) + } + + handler.called = true + } + } return nil } diff --git a/chain/events/events_test.go b/chain/events/events_test.go index 04f938055..61dd25fbb 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -41,50 +41,102 @@ type fakeCS struct { msgs map[cid.Cid]fakeMsg blkMsgs map[cid.Cid]cid.Cid - sync sync.Mutex - tipsets map[types.TipSetKey]*types.TipSet - sub func(rev, app []*types.TipSet) + mu sync.Mutex + waitSub chan struct{} + subCh chan<- []*api.HeadChange + callNumber map[string]int +} - callNumberLk sync.Mutex - callNumber map[string]int +func newFakeCS(t *testing.T) *fakeCS { + fcs := &fakeCS{ + t: t, + h: 1, + msgs: make(map[cid.Cid]fakeMsg), + blkMsgs: make(map[cid.Cid]cid.Cid), + tipsets: make(map[types.TipSetKey]*types.TipSet), + tsc: newTSCache(nil, 2*build.ForkLengthThreshold), + callNumber: map[string]int{}, + waitSub: make(chan struct{}), + } + require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + return fcs } func (fcs *fakeCS) ChainHead(ctx context.Context) (*types.TipSet, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainHead"] = fcs.callNumber["ChainHead"] + 1 panic("implement me") } +func (fcs *fakeCS) ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) { + fcs.mu.Lock() + fcs.callNumber["ChainGetPath"] = fcs.callNumber["ChainGetPath"] + 1 + fcs.mu.Unlock() + + fromTs, err := fcs.ChainGetTipSet(ctx, from) + if err != nil { + return nil, err + } + + toTs, err := fcs.ChainGetTipSet(ctx, to) + if err != nil { + return nil, err + } + + // copied from the chainstore + revert, apply, err := store.ReorgOps(func(tsk types.TipSetKey) (*types.TipSet, error) { + return fcs.ChainGetTipSet(ctx, tsk) + }, fromTs, toTs) + if err != nil { + return nil, err + } + + path := make([]*api.HeadChange, len(revert)+len(apply)) + for i, r := range revert { + path[i] = &api.HeadChange{Type: store.HCRevert, Val: r} + } + for j, i := 0, len(apply)-1; i >= 0; j, i = j+1, i-1 { + path[j+len(revert)] = &api.HeadChange{Type: store.HCApply, Val: apply[i]} + } + return path, nil +} + func (fcs *fakeCS) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainGetTipSet"] = fcs.callNumber["ChainGetTipSet"] + 1 return fcs.tipsets[key], nil } func (fcs *fakeCS) StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["StateSearchMsg"] = fcs.callNumber["StateSearchMsg"] + 1 return nil, nil } func (fcs *fakeCS) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["StateGetActor"] = fcs.callNumber["StateGetActor"] + 1 panic("Not Implemented") } func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainGetTipSetByHeight"] = fcs.callNumber["ChainGetTipSetByHeight"] + 1 panic("Not Implemented") } +func (fcs *fakeCS) ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) { + fcs.mu.Lock() + defer fcs.mu.Unlock() + fcs.callNumber["ChainGetTipSetAfterHeight"] = fcs.callNumber["ChainGetTipSetAfterHeight"] + 1 + panic("Not Implemented") +} func (fcs *fakeCS) makeTs(t *testing.T, parents []cid.Cid, h abi.ChainEpoch, msgcid cid.Cid) *types.TipSet { a, _ := address.NewFromString("t00") @@ -132,43 +184,33 @@ func (fcs *fakeCS) makeTs(t *testing.T, parents []cid.Cid, h abi.ChainEpoch, msg return ts } -func (fcs *fakeCS) ChainNotify(context.Context) (<-chan []*api.HeadChange, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() +func (fcs *fakeCS) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainNotify"] = fcs.callNumber["ChainNotify"] + 1 out := make(chan []*api.HeadChange, 1) - best, err := fcs.tsc.best() + if fcs.subCh != nil { + close(out) + fcs.t.Error("already subscribed to notifications") + return out, nil + } + + best, err := fcs.tsc.ChainHead(ctx) if err != nil { return nil, err } + out <- []*api.HeadChange{{Type: store.HCCurrent, Val: best}} - - fcs.sub = func(rev, app []*types.TipSet) { - notif := make([]*api.HeadChange, len(rev)+len(app)) - - for i, r := range rev { - notif[i] = &api.HeadChange{ - Type: store.HCRevert, - Val: r, - } - } - for i, r := range app { - notif[i+len(rev)] = &api.HeadChange{ - Type: store.HCApply, - Val: r, - } - } - - out <- notif - } + fcs.subCh = out + close(fcs.waitSub) return out, nil } func (fcs *fakeCS) ChainGetBlockMessages(ctx context.Context, blk cid.Cid) (*api.BlockMessages, error) { - fcs.callNumberLk.Lock() - defer fcs.callNumberLk.Unlock() + fcs.mu.Lock() + defer fcs.mu.Unlock() fcs.callNumber["ChainGetBlockMessages"] = fcs.callNumber["ChainGetBlockMessages"] + 1 messages, ok := fcs.blkMsgs[blk] if !ok { @@ -180,7 +222,15 @@ func (fcs *fakeCS) ChainGetBlockMessages(ctx context.Context, blk cid.Cid) (*api return &api.BlockMessages{}, nil } - return &api.BlockMessages{BlsMessages: ms.bmsgs, SecpkMessages: ms.smsgs}, nil + cids := make([]cid.Cid, len(ms.bmsgs)+len(ms.smsgs)) + for i, m := range ms.bmsgs { + cids[i] = m.Cid() + } + for i, m := range ms.smsgs { + cids[i+len(ms.bmsgs)] = m.Cid() + } + + return &api.BlockMessages{BlsMessages: ms.bmsgs, SecpkMessages: ms.smsgs, Cids: cids}, nil } func (fcs *fakeCS) fakeMsgs(m fakeMsg) cid.Cid { @@ -197,11 +247,47 @@ func (fcs *fakeCS) fakeMsgs(m fakeMsg) cid.Cid { return c } -func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { // todo: allow msgs - if fcs.sub == nil { +func (fcs *fakeCS) dropSub() { + fcs.mu.Lock() + + if fcs.subCh == nil { + fcs.mu.Unlock() fcs.t.Fatal("sub not be nil") } + waitCh := make(chan struct{}) + fcs.waitSub = waitCh + close(fcs.subCh) + fcs.subCh = nil + fcs.mu.Unlock() + + <-waitCh +} + +func (fcs *fakeCS) sub(rev, app []*types.TipSet) { + <-fcs.waitSub + notif := make([]*api.HeadChange, len(rev)+len(app)) + + for i, r := range rev { + notif[i] = &api.HeadChange{ + Type: store.HCRevert, + Val: r, + } + } + for i, r := range app { + notif[i+len(rev)] = &api.HeadChange{ + Type: store.HCApply, + Val: r, + } + } + + fcs.subCh <- notif +} + +func (fcs *fakeCS) advance(rev, app, drop int, msgs map[int]cid.Cid, nulls ...int) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nullm := map[int]struct{}{} for _, v := range nulls { nullm[v] = struct{}{} @@ -209,12 +295,22 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / var revs []*types.TipSet for i := 0; i < rev; i++ { - ts, err := fcs.tsc.best() + fcs.t.Log("revert", fcs.h) + from, err := fcs.tsc.ChainHead(ctx) require.NoError(fcs.t, err) - if _, ok := nullm[int(ts.Height())]; !ok { - revs = append(revs, ts) - require.NoError(fcs.t, fcs.tsc.revert(ts)) + if _, ok := nullm[int(from.Height())]; !ok { + require.NoError(fcs.t, fcs.tsc.revert(from)) + + if drop == 0 { + revs = append(revs, from) + } + } + if drop > 0 { + drop-- + if drop == 0 { + fcs.dropSub() + } } fcs.h-- } @@ -222,57 +318,54 @@ func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { / var apps []*types.TipSet for i := 0; i < app; i++ { fcs.h++ + fcs.t.Log("apply", fcs.h) mc, hasMsgs := msgs[i] if !hasMsgs { mc = dummyCid } - if _, ok := nullm[int(fcs.h)]; ok { - continue + if _, ok := nullm[int(fcs.h)]; !ok { + best, err := fcs.tsc.ChainHead(ctx) + require.NoError(fcs.t, err) + ts := fcs.makeTs(fcs.t, best.Key().Cids(), fcs.h, mc) + require.NoError(fcs.t, fcs.tsc.add(ts)) + + if hasMsgs { + fcs.blkMsgs[ts.Blocks()[0].Cid()] = mc + } + + if drop == 0 { + apps = append(apps, ts) + } } - best, err := fcs.tsc.best() - require.NoError(fcs.t, err) - ts := fcs.makeTs(fcs.t, best.Key().Cids(), fcs.h, mc) - require.NoError(fcs.t, fcs.tsc.add(ts)) - - if hasMsgs { - fcs.blkMsgs[ts.Blocks()[0].Cid()] = mc + if drop > 0 { + drop-- + if drop == 0 { + fcs.dropSub() + } } - - apps = append(apps, ts) } - fcs.sync.Lock() - fcs.sub(revs, apps) - fcs.sync.Lock() - fcs.sync.Unlock() //nolint:staticcheck -} - -func (fcs *fakeCS) notifDone() { - fcs.sync.Unlock() + // Wait for the last round to finish. + fcs.sub(nil, nil) + fcs.sub(nil, nil) } var _ EventAPI = &fakeCS{} func TestAt(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + fcs := newFakeCS(t) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true @@ -283,105 +376,56 @@ func TestAt(t *testing.T) { }, 3, 5) require.NoError(t, err) - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(10, 10, nil) + fcs.advance(10, 10, 0, nil) require.Equal(t, true, applied) require.Equal(t, true, reverted) applied = false reverted = false - fcs.advance(10, 1, nil) + fcs.advance(10, 1, 0, nil) require.Equal(t, false, applied) require.Equal(t, true, reverted) reverted = false - fcs.advance(0, 1, nil) + fcs.advance(0, 1, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 2, nil) + fcs.advance(0, 2, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 1, nil) // 8 + fcs.advance(0, 1, 0, nil) // 8 require.Equal(t, true, applied) require.Equal(t, false, reverted) } -func TestAtDoubleTrigger(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) - - var applied bool - var reverted bool - - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { - require.Equal(t, 5, int(ts.Height())) - require.Equal(t, 8, int(curH)) - applied = true - return nil - }, func(_ context.Context, ts *types.TipSet) error { - reverted = true - return nil - }, 3, 5) - require.NoError(t, err) - - fcs.advance(0, 6, nil) - require.False(t, applied) - require.False(t, reverted) - - fcs.advance(0, 1, nil) - require.True(t, applied) - require.False(t, reverted) - applied = false - - fcs.advance(2, 2, nil) - require.False(t, applied) - require.False(t, reverted) - - fcs.advance(4, 4, nil) - require.True(t, applied) - require.True(t, reverted) -} - func TestAtNullTrigger(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + fcs := newFakeCS(t) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, abi.ChainEpoch(6), ts.Height()) require.Equal(t, 8, int(curH)) applied = true @@ -392,31 +436,29 @@ func TestAtNullTrigger(t *testing.T) { }, 3, 5) require.NoError(t, err) - fcs.advance(0, 6, nil, 5) + fcs.advance(0, 6, 0, nil, 5) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false } func TestAtNullConf(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - events := NewEvents(context.Background(), fcs) + fcs := newFakeCS(t) + + events, err := NewEvents(ctx, fcs) + require.NoError(t, err) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(ctx, func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true @@ -427,38 +469,33 @@ func TestAtNullConf(t *testing.T) { }, 3, 5) require.NoError(t, err) - fcs.advance(0, 6, nil) + fcs.advance(0, 6, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 3, nil, 8) + fcs.advance(0, 3, 0, nil, 8) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false - fcs.advance(7, 1, nil) + fcs.advance(7, 1, 0, nil) require.Equal(t, false, applied) require.Equal(t, true, reverted) reverted = false } func TestAtStart(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) - fcs.advance(0, 5, nil) // 6 + fcs.advance(0, 5, 0, nil) // 6 var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true @@ -472,28 +509,23 @@ func TestAtStart(t *testing.T) { require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(0, 5, nil) // 11 + fcs.advance(0, 5, 0, nil) // 11 require.Equal(t, true, applied) require.Equal(t, false, reverted) } func TestAtStartConfidence(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) - fcs.advance(0, 10, nil) // 11 + fcs.advance(0, 10, 0, nil) // 11 var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 11, int(curH)) applied = true @@ -509,21 +541,16 @@ func TestAtStartConfidence(t *testing.T) { } func TestAtChained(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { - return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + return events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 10, int(ts.Height())) applied = true return nil @@ -537,30 +564,25 @@ func TestAtChained(t *testing.T) { }, 3, 5) require.NoError(t, err) - fcs.advance(0, 15, nil) + fcs.advance(0, 15, 0, nil) require.Equal(t, true, applied) require.Equal(t, false, reverted) } func TestAtChainedConfidence(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) - fcs.advance(0, 15, nil) + fcs.advance(0, 15, 0, nil) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { - return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + return events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { require.Equal(t, 10, int(ts.Height())) applied = true return nil @@ -579,22 +601,17 @@ func TestAtChainedConfidence(t *testing.T) { } func TestAtChainedConfidenceNull(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + fcs := newFakeCS(t) - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) - fcs.advance(0, 15, nil, 5) + fcs.advance(0, 15, 0, nil, 5) var applied bool var reverted bool - err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + err = events.ChainAt(context.Background(), func(_ context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { applied = true require.Equal(t, 6, int(ts.Height())) return nil @@ -615,18 +632,10 @@ func matchAddrMethod(to address.Address, m abi.MethodNum) func(msg *types.Messag } func TestCalled(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) @@ -637,7 +646,7 @@ func TestCalled(t *testing.T) { var appliedTs *types.TipSet var appliedH abi.ChainEpoch - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { require.Equal(t, false, applied) @@ -654,13 +663,13 @@ func TestCalled(t *testing.T) { // create few blocks to make sure nothing get's randomly called - fcs.advance(0, 4, nil) // H=5 + fcs.advance(0, 4, 0, nil) // H=5 require.Equal(t, false, applied) require.Equal(t, false, reverted) // create blocks with message (but below confidence threshold) - fcs.advance(0, 3, map[int]cid.Cid{ // msg at H=6; H=8 (confidence=2) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // msg at H=6; H=8 (confidence=2) 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -673,14 +682,14 @@ func TestCalled(t *testing.T) { // create additional block so we are above confidence threshold - fcs.advance(0, 2, nil) // H=10 (confidence=3, apply) + fcs.advance(0, 2, 0, nil) // H=10 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false // dip below confidence - fcs.advance(2, 2, nil) // H=10 (confidence=3, apply) + fcs.advance(2, 2, 0, nil) // H=10 (confidence=3, apply) require.Equal(t, false, applied) require.Equal(t, false, reverted) @@ -694,13 +703,13 @@ func TestCalled(t *testing.T) { // revert some blocks, keep the message - fcs.advance(3, 1, nil) // H=8 (confidence=1) + fcs.advance(3, 1, 0, nil) // H=8 (confidence=1) require.Equal(t, false, applied) require.Equal(t, false, reverted) // revert the message - fcs.advance(2, 1, nil) // H=7, we reverted ts with the msg execution, but not the msg itself + fcs.advance(2, 1, 0, nil) // H=7, we reverted ts with the msg execution, but not the msg itself require.Equal(t, false, applied) require.Equal(t, true, reverted) @@ -714,7 +723,7 @@ func TestCalled(t *testing.T) { }, }) - fcs.advance(0, 3, map[int]cid.Cid{ // (n2msg confidence=1) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // (n2msg confidence=1) 0: n2msg, }) @@ -723,7 +732,7 @@ func TestCalled(t *testing.T) { require.Equal(t, abi.ChainEpoch(10), appliedH) applied = false - fcs.advance(0, 2, nil) // (confidence=3) + fcs.advance(0, 2, 0, nil) // (confidence=3) require.Equal(t, true, applied) require.Equal(t, false, reverted) @@ -738,7 +747,7 @@ func TestCalled(t *testing.T) { // revert and apply at different height - fcs.advance(8, 6, map[int]cid.Cid{ // (confidence=3) + fcs.advance(8, 6, 0, map[int]cid.Cid{ // (confidence=3) 1: n2msg, }) @@ -759,7 +768,7 @@ func TestCalled(t *testing.T) { // call method again - fcs.advance(0, 5, map[int]cid.Cid{ + fcs.advance(0, 5, 0, map[int]cid.Cid{ 0: n2msg, }) @@ -768,7 +777,7 @@ func TestCalled(t *testing.T) { applied = false // send and revert below confidence, then cross confidence - fcs.advance(0, 2, map[int]cid.Cid{ + fcs.advance(0, 2, 0, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 3}, @@ -776,14 +785,14 @@ func TestCalled(t *testing.T) { }), }) - fcs.advance(2, 5, nil) // H=19, but message reverted + fcs.advance(2, 5, 0, nil) // H=19, but message reverted require.Equal(t, false, applied) require.Equal(t, false, reverted) // test timeout (it's set to 20 in the call to `events.Called` above) - fcs.advance(0, 6, nil) + fcs.advance(0, 6, 0, nil) require.Equal(t, false, applied) // not calling timeout as we received messages require.Equal(t, false, reverted) @@ -791,7 +800,7 @@ func TestCalled(t *testing.T) { // test unregistering with more more = false - fcs.advance(0, 5, map[int]cid.Cid{ + fcs.advance(0, 5, 0, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 4}, // this signals we don't want more @@ -803,7 +812,7 @@ func TestCalled(t *testing.T) { require.Equal(t, false, reverted) applied = false - fcs.advance(0, 5, map[int]cid.Cid{ + fcs.advance(0, 5, 0, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 5}, @@ -816,37 +825,29 @@ func TestCalled(t *testing.T) { // revert after disabled - fcs.advance(5, 1, nil) // try reverting msg sent after disabling + fcs.advance(5, 1, 0, nil) // try reverting msg sent after disabling require.Equal(t, false, applied) require.Equal(t, false, reverted) - fcs.advance(5, 1, nil) // try reverting msg sent before disabling + fcs.advance(5, 1, 0, nil) // try reverting msg sent before disabling require.Equal(t, false, applied) require.Equal(t, true, reverted) } func TestCalledTimeout(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) called := false - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { called = true @@ -860,29 +861,21 @@ func TestCalledTimeout(t *testing.T) { }, 3, 20, matchAddrMethod(t0123, 5)) require.NoError(t, err) - fcs.advance(0, 21, nil) + fcs.advance(0, 21, 0, nil) require.False(t, called) - fcs.advance(0, 5, nil) + fcs.advance(0, 5, 0, nil) require.True(t, called) called = false // with check func reporting done - fcs = &fakeCS{ - t: t, - h: 1, + fcs = newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - callNumber: map[string]int{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + events, err = NewEvents(context.Background(), fcs) + require.NoError(t, err) - events = NewEvents(context.Background(), fcs) - - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return true, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { called = true @@ -896,33 +889,25 @@ func TestCalledTimeout(t *testing.T) { }, 3, 20, matchAddrMethod(t0123, 5)) require.NoError(t, err) - fcs.advance(0, 21, nil) + fcs.advance(0, 21, 0, nil) require.False(t, called) - fcs.advance(0, 5, nil) + fcs.advance(0, 5, 0, nil) require.False(t, called) } func TestCalledOrder(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) at := 0 - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { switch at { @@ -951,7 +936,7 @@ func TestCalledOrder(t *testing.T) { }, 3, 20, matchAddrMethod(t0123, 5)) require.NoError(t, err) - fcs.advance(0, 10, map[int]cid.Cid{ + fcs.advance(0, 10, 0, map[int]cid.Cid{ 1: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -964,22 +949,14 @@ func TestCalledOrder(t *testing.T) { }), }) - fcs.advance(9, 1, nil) + fcs.advance(9, 1, 0, nil) } func TestCalledNull(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) @@ -987,7 +964,7 @@ func TestCalledNull(t *testing.T) { more := true var applied, reverted bool - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { require.Equal(t, false, applied) @@ -1001,13 +978,13 @@ func TestCalledNull(t *testing.T) { // create few blocks to make sure nothing get's randomly called - fcs.advance(0, 4, nil) // H=5 + fcs.advance(0, 4, 0, nil) // H=5 require.Equal(t, false, applied) require.Equal(t, false, reverted) // create blocks with message (but below confidence threshold) - fcs.advance(0, 3, map[int]cid.Cid{ // msg at H=6; H=8 (confidence=2) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // msg at H=6; H=8 (confidence=2) 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -1021,31 +998,23 @@ func TestCalledNull(t *testing.T) { // create additional blocks so we are above confidence threshold, but with null tipset at the height // of application - fcs.advance(0, 3, nil, 10) // H=11 (confidence=3, apply) + fcs.advance(0, 3, 0, nil, 10) // H=11 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false - fcs.advance(5, 1, nil, 10) + fcs.advance(5, 1, 0, nil, 10) require.Equal(t, false, applied) require.Equal(t, true, reverted) } func TestRemoveTriggersOnMessage(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) @@ -1053,7 +1022,7 @@ func TestRemoveTriggersOnMessage(t *testing.T) { more := true var applied, reverted bool - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { require.Equal(t, false, applied) @@ -1067,13 +1036,13 @@ func TestRemoveTriggersOnMessage(t *testing.T) { // create few blocks to make sure nothing get's randomly called - fcs.advance(0, 4, nil) // H=5 + fcs.advance(0, 4, 0, nil) // H=5 require.Equal(t, false, applied) require.Equal(t, false, reverted) // create blocks with message (but below confidence threshold) - fcs.advance(0, 3, map[int]cid.Cid{ // msg occurs at H=5, applied at H=6; H=8 (confidence=2) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // msg occurs at H=5, applied at H=6; H=8 (confidence=2) 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -1085,19 +1054,19 @@ func TestRemoveTriggersOnMessage(t *testing.T) { require.Equal(t, false, reverted) // revert applied TS & message TS - fcs.advance(3, 1, nil) // H=6 (tipset message applied in reverted, AND message reverted) + fcs.advance(3, 1, 0, nil) // H=6 (tipset message applied in reverted, AND message reverted) require.Equal(t, false, applied) require.Equal(t, false, reverted) // create additional blocks so we are above confidence threshold, but message not applied // as it was reverted - fcs.advance(0, 5, nil) // H=11 (confidence=3, apply) + fcs.advance(0, 5, 0, nil) // H=11 (confidence=3, apply) require.Equal(t, false, applied) require.Equal(t, false, reverted) // create blocks with message again (but below confidence threshold) - fcs.advance(0, 3, map[int]cid.Cid{ // msg occurs at H=12, applied at H=13; H=15 (confidence=2) + fcs.advance(0, 3, 0, map[int]cid.Cid{ // msg occurs at H=12, applied at H=13; H=15 (confidence=2) 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 2}, @@ -1108,12 +1077,12 @@ func TestRemoveTriggersOnMessage(t *testing.T) { require.Equal(t, false, reverted) // revert applied height TS, but don't remove message trigger - fcs.advance(2, 1, nil) // H=13 (tipset message applied in reverted, by tipset with message not reverted) + fcs.advance(2, 1, 0, nil) // H=13 (tipset message applied in reverted, by tipset with message not reverted) require.Equal(t, false, applied) require.Equal(t, false, reverted) // create additional blocks so we are above confidence threshold - fcs.advance(0, 4, nil) // H=18 (confidence=3, apply) + fcs.advance(0, 4, 0, nil) // H=18 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) @@ -1125,18 +1094,10 @@ type testStateChange struct { } func TestStateChanged(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) more := true var applied, reverted bool @@ -1149,9 +1110,12 @@ func TestStateChanged(t *testing.T) { confidence := 3 timeout := abi.ChainEpoch(20) - err := events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { + if data != nil { + require.Equal(t, oldTs.Key(), newTs.Parents()) + } require.Equal(t, false, applied) applied = true appliedData = data @@ -1163,6 +1127,7 @@ func TestStateChanged(t *testing.T) { reverted = true return nil }, confidence, timeout, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { + require.Equal(t, oldTs.Key(), newTs.Parents()) if matchData == nil { return false, matchData, nil } @@ -1175,27 +1140,27 @@ func TestStateChanged(t *testing.T) { // create few blocks to make sure nothing get's randomly called - fcs.advance(0, 4, nil) // H=5 + fcs.advance(0, 4, 0, nil) // H=5 require.Equal(t, false, applied) require.Equal(t, false, reverted) // create state change (but below confidence threshold) matchData = testStateChange{from: "a", to: "b"} - fcs.advance(0, 3, nil) + fcs.advance(0, 3, 0, nil) require.Equal(t, false, applied) require.Equal(t, false, reverted) // create additional block so we are above confidence threshold - fcs.advance(0, 2, nil) // H=10 (confidence=3, apply) + fcs.advance(0, 2, 0, nil) // H=10 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false // dip below confidence (should not apply again) - fcs.advance(2, 2, nil) // H=10 (confidence=3, apply) + fcs.advance(2, 2, 0, nil) // H=10 (confidence=3, apply) require.Equal(t, false, applied) require.Equal(t, false, reverted) @@ -1214,18 +1179,10 @@ func TestStateChanged(t *testing.T) { } func TestStateChangedRevert(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) more := true var applied, reverted bool @@ -1234,9 +1191,12 @@ func TestStateChangedRevert(t *testing.T) { confidence := 1 timeout := abi.ChainEpoch(20) - err := events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.StateChanged(func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { + if data != nil { + require.Equal(t, oldTs.Key(), newTs.Parents()) + } require.Equal(t, false, applied) applied = true return more, nil @@ -1244,6 +1204,8 @@ func TestStateChangedRevert(t *testing.T) { reverted = true return nil }, confidence, timeout, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { + require.Equal(t, oldTs.Key(), newTs.Parents()) + if matchData == nil { return false, matchData, nil } @@ -1254,18 +1216,18 @@ func TestStateChangedRevert(t *testing.T) { }) require.NoError(t, err) - fcs.advance(0, 2, nil) // H=3 + fcs.advance(0, 2, 0, nil) // H=3 // Make a state change from TS at height 3 to TS at height 4 matchData = testStateChange{from: "a", to: "b"} - fcs.advance(0, 1, nil) // H=4 + fcs.advance(0, 1, 0, nil) // H=4 // Haven't yet reached confidence require.Equal(t, false, applied) require.Equal(t, false, reverted) // Advance to reach confidence level - fcs.advance(0, 1, nil) // H=5 + fcs.advance(0, 1, 0, nil) // H=5 // Should now have called the handler require.Equal(t, true, applied) @@ -1273,19 +1235,19 @@ func TestStateChangedRevert(t *testing.T) { applied = false // Advance 3 more TS - fcs.advance(0, 3, nil) // H=8 + fcs.advance(0, 3, 0, nil) // H=8 require.Equal(t, false, applied) require.Equal(t, false, reverted) // Regress but not so far as to cause a revert - fcs.advance(3, 1, nil) // H=6 + fcs.advance(3, 1, 0, nil) // H=6 require.Equal(t, false, applied) require.Equal(t, false, reverted) // Regress back to state where change happened - fcs.advance(3, 1, nil) // H=4 + fcs.advance(3, 1, 0, nil) // H=4 // Expect revert to have happened require.Equal(t, false, applied) @@ -1293,103 +1255,94 @@ func TestStateChangedRevert(t *testing.T) { } func TestStateChangedTimeout(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + timeoutHeight := abi.ChainEpoch(20) + confidence := 3 - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - callNumber: map[string]int{}, + testCases := []struct { + name string + checkFn CheckFunc + nilBlocks []int + expectTimeout bool + }{{ + // Verify that the state changed timeout is called at the expected height + name: "state changed timeout", + checkFn: func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { + return false, true, nil + }, + expectTimeout: true, + }, { + // Verify that the state changed timeout is called even if the timeout + // falls on nil block + name: "state changed timeout falls on nil block", + checkFn: func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { + return false, true, nil + }, + nilBlocks: []int{20, 21, 22, 23}, + expectTimeout: true, + }, { + // Verify that the state changed timeout is not called if the check + // function reports that it's complete + name: "no timeout callback if check func reports done", + checkFn: func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { + return true, true, nil + }, + expectTimeout: false, + }} + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + fcs := newFakeCS(t) + + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) + + // Track whether the callback was called + called := false + + // Set up state change tracking that will timeout at the given height + err = events.StateChanged( + tc.checkFn, + func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { + // Expect the callback to be called at the timeout height with nil data + called = true + require.Nil(t, data) + require.Equal(t, timeoutHeight, newTs.Height()) + require.Equal(t, timeoutHeight+abi.ChainEpoch(confidence), curH) + return false, nil + }, func(_ context.Context, ts *types.TipSet) error { + t.Fatal("revert on timeout") + return nil + }, confidence, timeoutHeight, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { + return false, nil, nil + }) + + require.NoError(t, err) + + // Advance to timeout height + fcs.advance(0, int(timeoutHeight)+1, 0, nil) + require.False(t, called) + + // Advance past timeout height + fcs.advance(0, 5, 0, nil, tc.nilBlocks...) + require.Equal(t, tc.expectTimeout, called) + called = false + }) } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) - - called := false - - err := events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) { - return false, true, nil - }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { - called = true - require.Nil(t, data) - require.Equal(t, abi.ChainEpoch(20), newTs.Height()) - require.Equal(t, abi.ChainEpoch(23), curH) - return false, nil - }, func(_ context.Context, ts *types.TipSet) error { - t.Fatal("revert on timeout") - return nil - }, 3, 20, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { - return false, nil, nil - }) - - require.NoError(t, err) - - fcs.advance(0, 21, nil) - require.False(t, called) - - fcs.advance(0, 5, nil) - require.True(t, called) - called = false - - // with check func reporting done - - fcs = &fakeCS{ - t: t, - h: 1, - - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - callNumber: map[string]int{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events = NewEvents(context.Background(), fcs) - - err = events.StateChanged(func(ts *types.TipSet) (d bool, m bool, e error) { - return true, true, nil - }, func(oldTs, newTs *types.TipSet, data StateChange, curH abi.ChainEpoch) (bool, error) { - called = true - require.Nil(t, data) - require.Equal(t, abi.ChainEpoch(20), newTs.Height()) - require.Equal(t, abi.ChainEpoch(23), curH) - return false, nil - }, func(_ context.Context, ts *types.TipSet) error { - t.Fatal("revert on timeout") - return nil - }, 3, 20, func(oldTs, newTs *types.TipSet) (bool, StateChange, error) { - return false, nil, nil - }) - require.NoError(t, err) - - fcs.advance(0, 21, nil) - require.False(t, called) - - fcs.advance(0, 5, nil) - require.False(t, called) } func TestCalledMultiplePerEpoch(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - callNumber: map[string]int{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) - - events := NewEvents(context.Background(), fcs) + events, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) t0123, err := address.NewFromString("t0123") require.NoError(t, err) at := 0 - err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { + err = events.Called(context.Background(), func(ctx context.Context, ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (bool, error) { switch at { @@ -1418,7 +1371,7 @@ func TestCalledMultiplePerEpoch(t *testing.T) { }, 3, 20, matchAddrMethod(t0123, 5)) require.NoError(t, err) - fcs.advance(0, 10, map[int]cid.Cid{ + fcs.advance(0, 10, 0, map[int]cid.Cid{ 1: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 1}, @@ -1427,26 +1380,72 @@ func TestCalledMultiplePerEpoch(t *testing.T) { }), }) - fcs.advance(9, 1, nil) + fcs.advance(9, 1, 0, nil) } func TestCachedSameBlock(t *testing.T) { - fcs := &fakeCS{ - t: t, - h: 1, + fcs := newFakeCS(t) - msgs: map[cid.Cid]fakeMsg{}, - blkMsgs: map[cid.Cid]cid.Cid{}, - callNumber: map[string]int{}, - tsc: newTSCache(2*build.ForkLengthThreshold, nil), - } - require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid))) + _, err := NewEvents(context.Background(), fcs) + require.NoError(t, err) - _ = NewEvents(context.Background(), fcs) - - fcs.advance(0, 10, map[int]cid.Cid{}) + fcs.advance(0, 10, 0, map[int]cid.Cid{}) assert.Assert(t, fcs.callNumber["ChainGetBlockMessages"] == 20, "expect call ChainGetBlockMessages %d but got ", 20, fcs.callNumber["ChainGetBlockMessages"]) - fcs.advance(5, 10, map[int]cid.Cid{}) + fcs.advance(5, 10, 0, map[int]cid.Cid{}) assert.Assert(t, fcs.callNumber["ChainGetBlockMessages"] == 30, "expect call ChainGetBlockMessages %d but got ", 30, fcs.callNumber["ChainGetBlockMessages"]) } + +type testObserver struct { + t *testing.T + head *types.TipSet +} + +func (t *testObserver) Apply(_ context.Context, from, to *types.TipSet) error { + if t.head != nil { + require.True(t.t, t.head.Equals(from)) + } + t.head = to + return nil +} + +func (t *testObserver) Revert(_ context.Context, from, to *types.TipSet) error { + if t.head != nil { + require.True(t.t, t.head.Equals(from)) + } + t.head = to + return nil +} + +func TestReconnect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fcs := newFakeCS(t) + + events, err := NewEvents(ctx, fcs) + require.NoError(t, err) + + fcs.advance(0, 1, 0, nil) + + events.Observe(&testObserver{t: t}) + + fcs.advance(0, 3, 0, nil) + + // Drop on apply + fcs.advance(0, 6, 2, nil) + require.True(t, fcs.callNumber["ChainGetPath"] == 1) + + // drop across revert/apply boundary + fcs.advance(4, 2, 3, nil) + require.True(t, fcs.callNumber["ChainGetPath"] == 2) + fcs.advance(0, 6, 0, nil) + + // drop on revert + fcs.advance(3, 0, 2, nil) + require.True(t, fcs.callNumber["ChainGetPath"] == 3) + + // drop with nulls + fcs.advance(0, 5, 2, nil, 0, 1, 3) + require.True(t, fcs.callNumber["ChainGetPath"] == 4) +} diff --git a/chain/events/message_cache.go b/chain/events/message_cache.go new file mode 100644 index 000000000..75e179ad9 --- /dev/null +++ b/chain/events/message_cache.go @@ -0,0 +1,42 @@ +package events + +import ( + "context" + "sync" + + "github.com/filecoin-project/lotus/api" + lru "github.com/hashicorp/golang-lru" + "github.com/ipfs/go-cid" +) + +type messageCache struct { + api EventAPI + + blockMsgLk sync.Mutex + blockMsgCache *lru.ARCCache +} + +func newMessageCache(api EventAPI) *messageCache { + blsMsgCache, _ := lru.NewARC(500) + + return &messageCache{ + api: api, + blockMsgCache: blsMsgCache, + } +} + +func (c *messageCache) ChainGetBlockMessages(ctx context.Context, blkCid cid.Cid) (*api.BlockMessages, error) { + c.blockMsgLk.Lock() + defer c.blockMsgLk.Unlock() + + msgsI, ok := c.blockMsgCache.Get(blkCid) + var err error + if !ok { + msgsI, err = c.api.ChainGetBlockMessages(ctx, blkCid) + if err != nil { + return nil, err + } + c.blockMsgCache.Add(blkCid, msgsI) + } + return msgsI.(*api.BlockMessages), nil +} diff --git a/chain/events/observer.go b/chain/events/observer.go new file mode 100644 index 000000000..c67d821b5 --- /dev/null +++ b/chain/events/observer.go @@ -0,0 +1,255 @@ +package events + +import ( + "context" + "sync" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "go.opencensus.io/trace" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" +) + +type observer struct { + api EventAPI + + gcConfidence abi.ChainEpoch + + ready chan struct{} + + lk sync.Mutex + head *types.TipSet + maxHeight abi.ChainEpoch + observers []TipSetObserver +} + +func newObserver(api *cache, gcConfidence abi.ChainEpoch) *observer { + obs := &observer{ + api: api, + gcConfidence: gcConfidence, + + ready: make(chan struct{}), + observers: []TipSetObserver{}, + } + obs.Observe(api.observer()) + return obs +} + +func (o *observer) start(ctx context.Context) error { + go o.listenHeadChanges(ctx) + + // Wait for the first tipset to be seen or bail if shutting down + select { + case <-o.ready: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (o *observer) listenHeadChanges(ctx context.Context) { + for { + if err := o.listenHeadChangesOnce(ctx); err != nil { + log.Errorf("listen head changes errored: %s", err) + } else { + log.Warn("listenHeadChanges quit") + } + select { + case <-build.Clock.After(time.Second): + case <-ctx.Done(): + log.Warnf("not restarting listenHeadChanges: context error: %s", ctx.Err()) + return + } + + log.Info("restarting listenHeadChanges") + } +} + +func (o *observer) listenHeadChangesOnce(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + notifs, err := o.api.ChainNotify(ctx) + if err != nil { + // Retry is handled by caller + return xerrors.Errorf("listenHeadChanges ChainNotify call failed: %w", err) + } + + var cur []*api.HeadChange + var ok bool + + // Wait for first tipset or bail + select { + case cur, ok = <-notifs: + if !ok { + return xerrors.Errorf("notification channel closed") + } + case <-ctx.Done(): + return ctx.Err() + } + + if len(cur) != 1 { + return xerrors.Errorf("unexpected initial head notification length: %d", len(cur)) + } + + if cur[0].Type != store.HCCurrent { + return xerrors.Errorf("expected first head notification type to be 'current', was '%s'", cur[0].Type) + } + + curHead := cur[0].Val + + o.lk.Lock() + if o.head == nil { + o.head = curHead + close(o.ready) + } + startHead := o.head + o.lk.Unlock() + + if !startHead.Equals(curHead) { + changes, err := o.api.ChainGetPath(ctx, startHead.Key(), curHead.Key()) + if err != nil { + return xerrors.Errorf("failed to get path from last applied tipset to head: %w", err) + } + + if err := o.applyChanges(ctx, changes); err != nil { + return xerrors.Errorf("failed catch-up head changes: %w", err) + } + } + + for changes := range notifs { + if err := o.applyChanges(ctx, changes); err != nil { + return err + } + } + + return nil +} + +func (o *observer) applyChanges(ctx context.Context, changes []*api.HeadChange) error { + // Used to wait for a prior notification round to finish (by tests) + if len(changes) == 0 { + return nil + } + + var rev, app []*types.TipSet + for _, changes := range changes { + switch changes.Type { + case store.HCRevert: + rev = append(rev, changes.Val) + case store.HCApply: + app = append(app, changes.Val) + default: + log.Errorf("unexpected head change notification type: '%s'", changes.Type) + } + } + + if err := o.headChange(ctx, rev, app); err != nil { + return xerrors.Errorf("failed to apply head changes: %w", err) + } + return nil +} + +func (o *observer) headChange(ctx context.Context, rev, app []*types.TipSet) error { + ctx, span := trace.StartSpan(ctx, "events.HeadChange") + span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev)))) + span.AddAttributes(trace.Int64Attribute("applies", int64(len(app)))) + + o.lk.Lock() + head := o.head + o.lk.Unlock() + + defer func() { + span.AddAttributes(trace.Int64Attribute("endHeight", int64(head.Height()))) + span.End() + }() + + // NOTE: bailing out here if the head isn't what we expected is fine. We'll re-start the + // entire process and handle any strange reorgs. + for i, from := range rev { + if !from.Equals(head) { + return xerrors.Errorf( + "expected to revert %s (%d), reverting %s (%d)", + head.Key(), head.Height(), from.Key(), from.Height(), + ) + } + var to *types.TipSet + if i+1 < len(rev) { + // If we have more reverts, the next revert is the next head. + to = rev[i+1] + } else { + // At the end of the revert sequenece, we need to lookup the joint tipset + // between the revert sequence and the apply sequence. + var err error + to, err = o.api.ChainGetTipSet(ctx, from.Parents()) + if err != nil { + // Well, this sucks. We'll bail and restart. + return xerrors.Errorf("failed to get tipset when reverting due to a SetHeead: %w", err) + } + } + + // Get the current observers and atomically set the head. + // + // 1. We need to get the observers every time in case some registered/deregistered. + // 2. We need to atomically set the head so new observers don't see events twice or + // skip them. + o.lk.Lock() + observers := o.observers + o.head = to + o.lk.Unlock() + + for _, obs := range observers { + if err := obs.Revert(ctx, from, to); err != nil { + log.Errorf("observer %T failed to apply tipset %s (%d) with: %s", obs, from.Key(), from.Height(), err) + } + } + + if to.Height() < o.maxHeight-o.gcConfidence { + log.Errorf("reverted past finality, from %d to %d", o.maxHeight, to.Height()) + } + + head = to + } + + for _, to := range app { + if to.Parents() != head.Key() { + return xerrors.Errorf( + "cannot apply %s (%d) with parents %s on top of %s (%d)", + to.Key(), to.Height(), to.Parents(), head.Key(), head.Height(), + ) + } + + o.lk.Lock() + observers := o.observers + o.head = to + o.lk.Unlock() + + for _, obs := range observers { + if err := obs.Apply(ctx, head, to); err != nil { + log.Errorf("observer %T failed to revert tipset %s (%d) with: %s", obs, to.Key(), to.Height(), err) + } + } + if to.Height() > o.maxHeight { + o.maxHeight = to.Height() + } + + head = to + } + return nil +} + +// Observe registers the observer, and returns the current tipset. The observer is guaranteed to +// observe events starting at this tipset. +// +// Returns nil if the observer hasn't started yet (but still registers). +func (o *observer) Observe(obs TipSetObserver) *types.TipSet { + o.lk.Lock() + defer o.lk.Unlock() + o.observers = append(o.observers, obs) + return o.head +} diff --git a/chain/events/tscache.go b/chain/events/tscache.go index 44699684e..033c72a22 100644 --- a/chain/events/tscache.go +++ b/chain/events/tscache.go @@ -11,7 +11,9 @@ import ( ) type tsCacheAPI interface { + ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) + ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error) ChainHead(context.Context) (*types.TipSet, error) } @@ -20,61 +22,157 @@ type tsCacheAPI interface { type tipSetCache struct { mu sync.RWMutex - cache []*types.TipSet - start int - len int + byKey map[types.TipSetKey]*types.TipSet + byHeight []*types.TipSet + start int // chain head (end) + len int storage tsCacheAPI } -func newTSCache(cap abi.ChainEpoch, storage tsCacheAPI) *tipSetCache { +func newTSCache(storage tsCacheAPI, cap abi.ChainEpoch) *tipSetCache { return &tipSetCache{ - cache: make([]*types.TipSet, cap), - start: 0, - len: 0, + byKey: make(map[types.TipSetKey]*types.TipSet, cap), + byHeight: make([]*types.TipSet, cap), + start: 0, + len: 0, storage: storage, } } +func (tsc *tipSetCache) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + if ts, ok := tsc.byKey[tsk]; ok { + return ts, nil + } + return tsc.storage.ChainGetTipSet(ctx, tsk) +} -func (tsc *tipSetCache) add(ts *types.TipSet) error { +func (tsc *tipSetCache) ChainGetTipSetByHeight(ctx context.Context, height abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + return tsc.get(ctx, height, tsk, true) +} + +func (tsc *tipSetCache) ChainGetTipSetAfterHeight(ctx context.Context, height abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + return tsc.get(ctx, height, tsk, false) +} + +func (tsc *tipSetCache) get(ctx context.Context, height abi.ChainEpoch, tsk types.TipSetKey, prev bool) (*types.TipSet, error) { + fallback := tsc.storage.ChainGetTipSetAfterHeight + if prev { + fallback = tsc.storage.ChainGetTipSetByHeight + } + tsc.mu.RLock() + + // Nothing in the cache? + if tsc.len == 0 { + tsc.mu.RUnlock() + log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height) + return fallback(ctx, height, tsk) + } + + // Resolve the head. + head := tsc.byHeight[tsc.start] + if !tsk.IsEmpty() { + // Not on this chain? + var ok bool + head, ok = tsc.byKey[tsk] + if !ok { + tsc.mu.RUnlock() + return fallback(ctx, height, tsk) + } + } + + headH := head.Height() + tailH := headH - abi.ChainEpoch(tsc.len) + + if headH == height { + tsc.mu.RUnlock() + return head, nil + } else if headH < height { + tsc.mu.RUnlock() + // If the user doesn't pass a tsk, we assume "head" is the last tipset we processed. + return nil, xerrors.Errorf("requested epoch is in the future") + } else if height < tailH { + log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tailH) + tsc.mu.RUnlock() + return fallback(ctx, height, head.Key()) + } + + direction := 1 + if prev { + direction = -1 + } + var ts *types.TipSet + for i := 0; i < tsc.len && ts == nil; i += direction { + ts = tsc.byHeight[normalModulo(tsc.start-int(headH-height)+i, len(tsc.byHeight))] + } + tsc.mu.RUnlock() + return ts, nil +} + +func (tsc *tipSetCache) ChainHead(ctx context.Context) (*types.TipSet, error) { + tsc.mu.RLock() + best := tsc.byHeight[tsc.start] + tsc.mu.RUnlock() + if best == nil { + return tsc.storage.ChainHead(ctx) + } + return best, nil +} + +func (tsc *tipSetCache) add(to *types.TipSet) error { tsc.mu.Lock() defer tsc.mu.Unlock() if tsc.len > 0 { - if tsc.cache[tsc.start].Height() >= ts.Height() { - return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.cache[tsc.start].Height()+1, ts.Height()) + best := tsc.byHeight[tsc.start] + if best.Height() >= to.Height() { + return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.byHeight[tsc.start].Height()+1, to.Height()) + } + if best.Key() != to.Parents() { + return xerrors.Errorf( + "tipSetCache.add: expected new tipset %s (%d) to follow %s (%d), its parents are %s", + to.Key(), to.Height(), best.Key(), best.Height(), best.Parents(), + ) } } - nextH := ts.Height() + nextH := to.Height() if tsc.len > 0 { - nextH = tsc.cache[tsc.start].Height() + 1 + nextH = tsc.byHeight[tsc.start].Height() + 1 } // fill null blocks - for nextH != ts.Height() { - tsc.start = normalModulo(tsc.start+1, len(tsc.cache)) - tsc.cache[tsc.start] = nil - if tsc.len < len(tsc.cache) { + for nextH != to.Height() { + tsc.start = normalModulo(tsc.start+1, len(tsc.byHeight)) + was := tsc.byHeight[tsc.start] + if was != nil { + tsc.byHeight[tsc.start] = nil + delete(tsc.byKey, was.Key()) + } + if tsc.len < len(tsc.byHeight) { tsc.len++ } nextH++ } - tsc.start = normalModulo(tsc.start+1, len(tsc.cache)) - tsc.cache[tsc.start] = ts - if tsc.len < len(tsc.cache) { + tsc.start = normalModulo(tsc.start+1, len(tsc.byHeight)) + was := tsc.byHeight[tsc.start] + if was != nil { + delete(tsc.byKey, was.Key()) + } + tsc.byHeight[tsc.start] = to + if tsc.len < len(tsc.byHeight) { tsc.len++ } + tsc.byKey[to.Key()] = to return nil } -func (tsc *tipSetCache) revert(ts *types.TipSet) error { +func (tsc *tipSetCache) revert(from *types.TipSet) error { tsc.mu.Lock() defer tsc.mu.Unlock() - return tsc.revertUnlocked(ts) + return tsc.revertUnlocked(from) } func (tsc *tipSetCache) revertUnlocked(ts *types.TipSet) error { @@ -82,75 +180,35 @@ func (tsc *tipSetCache) revertUnlocked(ts *types.TipSet) error { return nil // this can happen, and it's fine } - if !tsc.cache[tsc.start].Equals(ts) { + was := tsc.byHeight[tsc.start] + + if !was.Equals(ts) { return xerrors.New("tipSetCache.revert: revert tipset didn't match cache head") } + delete(tsc.byKey, was.Key()) - tsc.cache[tsc.start] = nil - tsc.start = normalModulo(tsc.start-1, len(tsc.cache)) + tsc.byHeight[tsc.start] = nil + tsc.start = normalModulo(tsc.start-1, len(tsc.byHeight)) tsc.len-- _ = tsc.revertUnlocked(nil) // revert null block gap return nil } -func (tsc *tipSetCache) getNonNull(height abi.ChainEpoch) (*types.TipSet, error) { - for { - ts, err := tsc.get(height) - if err != nil { - return nil, err - } - if ts != nil { - return ts, nil - } - height++ - } +func (tsc *tipSetCache) observer() TipSetObserver { + return (*tipSetCacheObserver)(tsc) } -func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { - tsc.mu.RLock() +type tipSetCacheObserver tipSetCache - if tsc.len == 0 { - tsc.mu.RUnlock() - log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height) - return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, types.EmptyTSK) - } +var _ TipSetObserver = new(tipSetCacheObserver) - headH := tsc.cache[tsc.start].Height() - - if height > headH { - tsc.mu.RUnlock() - return nil, xerrors.Errorf("tipSetCache.get: requested tipset not in cache (req: %d, cache head: %d)", height, headH) - } - - clen := len(tsc.cache) - var tail *types.TipSet - for i := 1; i <= tsc.len; i++ { - tail = tsc.cache[normalModulo(tsc.start-tsc.len+i, clen)] - if tail != nil { - break - } - } - - if height < tail.Height() { - tsc.mu.RUnlock() - log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tail.Height()) - return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, tail.Key()) - } - - ts := tsc.cache[normalModulo(tsc.start-int(headH-height), clen)] - tsc.mu.RUnlock() - return ts, nil +func (tsc *tipSetCacheObserver) Apply(_ context.Context, _, to *types.TipSet) error { + return (*tipSetCache)(tsc).add(to) } -func (tsc *tipSetCache) best() (*types.TipSet, error) { - tsc.mu.RLock() - best := tsc.cache[tsc.start] - tsc.mu.RUnlock() - if best == nil { - return tsc.storage.ChainHead(context.TODO()) - } - return best, nil +func (tsc *tipSetCacheObserver) Revert(ctx context.Context, from, _ *types.TipSet) error { + return (*tipSetCache)(tsc).revert(from) } func normalModulo(n, m int) int { diff --git a/chain/events/tscache_test.go b/chain/events/tscache_test.go index ab6336f24..c3779eb9e 100644 --- a/chain/events/tscache_test.go +++ b/chain/events/tscache_test.go @@ -6,61 +6,22 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" ) -func TestTsCache(t *testing.T) { - tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t}) - - h := abi.ChainEpoch(75) - - a, _ := address.NewFromString("t00") - - add := func() { - ts, err := types.NewTipSet([]*types.BlockHeader{{ - Miner: a, - Height: h, - ParentStateRoot: dummyCid, - Messages: dummyCid, - ParentMessageReceipts: dummyCid, - BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, - BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, - }}) - if err != nil { - t.Fatal(err) - } - if err := tsc.add(ts); err != nil { - t.Fatal(err) - } - h++ - } - - for i := 0; i < 9000; i++ { - if i%90 > 60 { - best, err := tsc.best() - if err != nil { - t.Fatal(err, "; i:", i) - return - } - if err := tsc.revert(best); err != nil { - t.Fatal(err, "; i:", i) - return - } - h-- - } else { - add() - } - } - -} - type tsCacheAPIFailOnStorageCall struct { t *testing.T } +func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSetAfterHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { + tc.t.Fatal("storage call") + return &types.TipSet{}, nil +} + func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { tc.t.Fatal("storage call") return &types.TipSet{}, nil @@ -69,100 +30,181 @@ func (tc *tsCacheAPIFailOnStorageCall) ChainHead(ctx context.Context) (*types.Ti tc.t.Fatal("storage call") return &types.TipSet{}, nil } +func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + tc.t.Fatal("storage call") + return &types.TipSet{}, nil +} + +type cacheHarness struct { + t *testing.T + + miner address.Address + tsc *tipSetCache + height abi.ChainEpoch +} + +func newCacheharness(t *testing.T) *cacheHarness { + a, err := address.NewFromString("t00") + require.NoError(t, err) + + h := &cacheHarness{ + t: t, + tsc: newTSCache(&tsCacheAPIFailOnStorageCall{t: t}, 50), + height: 75, + miner: a, + } + h.addWithParents(nil) + return h +} + +func (h *cacheHarness) addWithParents(parents []cid.Cid) { + ts, err := types.NewTipSet([]*types.BlockHeader{{ + Miner: h.miner, + Height: h.height, + ParentStateRoot: dummyCid, + Messages: dummyCid, + ParentMessageReceipts: dummyCid, + BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, + BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, + Parents: parents, + }}) + require.NoError(h.t, err) + require.NoError(h.t, h.tsc.add(ts)) + h.height++ +} + +func (h *cacheHarness) add() { + last, err := h.tsc.ChainHead(context.Background()) + require.NoError(h.t, err) + h.addWithParents(last.Cids()) +} + +func (h *cacheHarness) revert() { + best, err := h.tsc.ChainHead(context.Background()) + require.NoError(h.t, err) + err = h.tsc.revert(best) + require.NoError(h.t, err) + h.height-- +} + +func (h *cacheHarness) skip(n abi.ChainEpoch) { + h.height += n +} + +func TestTsCache(t *testing.T) { + h := newCacheharness(t) + + for i := 0; i < 9000; i++ { + if i%90 > 60 { + h.revert() + } else { + h.add() + } + } +} func TestTsCacheNulls(t *testing.T) { - tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t}) + ctx := context.Background() + h := newCacheharness(t) - h := abi.ChainEpoch(75) + h.add() + h.add() + h.add() + h.skip(5) - a, _ := address.NewFromString("t00") - add := func() { - ts, err := types.NewTipSet([]*types.BlockHeader{{ - Miner: a, - Height: h, - ParentStateRoot: dummyCid, - Messages: dummyCid, - ParentMessageReceipts: dummyCid, - BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, - BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, - }}) - if err != nil { - t.Fatal(err) - } - if err := tsc.add(ts); err != nil { - t.Fatal(err) - } - h++ - } + h.add() + h.add() - add() - add() - add() - h += 5 - - add() - add() - - best, err := tsc.best() + best, err := h.tsc.ChainHead(ctx) require.NoError(t, err) - require.Equal(t, h-1, best.Height()) + require.Equal(t, h.height-1, best.Height()) - ts, err := tsc.get(h - 1) + ts, err := h.tsc.ChainGetTipSetByHeight(ctx, h.height-1, types.EmptyTSK) require.NoError(t, err) - require.Equal(t, h-1, ts.Height()) + require.Equal(t, h.height-1, ts.Height()) - ts, err = tsc.get(h - 2) + ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-2, types.EmptyTSK) require.NoError(t, err) - require.Equal(t, h-2, ts.Height()) + require.Equal(t, h.height-2, ts.Height()) - ts, err = tsc.get(h - 3) + // Should skip the nulls and walk back to the last tipset. + ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-3, types.EmptyTSK) require.NoError(t, err) - require.Nil(t, ts) + require.Equal(t, h.height-8, ts.Height()) - ts, err = tsc.get(h - 8) + ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-8, types.EmptyTSK) require.NoError(t, err) - require.Equal(t, h-8, ts.Height()) + require.Equal(t, h.height-8, ts.Height()) - best, err = tsc.best() + best, err = h.tsc.ChainHead(ctx) require.NoError(t, err) - require.NoError(t, tsc.revert(best)) + require.NoError(t, h.tsc.revert(best)) - best, err = tsc.best() + best, err = h.tsc.ChainHead(ctx) require.NoError(t, err) - require.NoError(t, tsc.revert(best)) + require.NoError(t, h.tsc.revert(best)) - best, err = tsc.best() + best, err = h.tsc.ChainHead(ctx) require.NoError(t, err) - require.Equal(t, h-8, best.Height()) + require.Equal(t, h.height-8, best.Height()) - h += 50 - add() + h.skip(50) + h.add() - ts, err = tsc.get(h - 1) + ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-1, types.EmptyTSK) require.NoError(t, err) - require.Equal(t, h-1, ts.Height()) + require.Equal(t, h.height-1, ts.Height()) } type tsCacheAPIStorageCallCounter struct { - t *testing.T - chainGetTipSetByHeight int - chainHead int + t *testing.T + chainGetTipSetByHeight int + chainGetTipSetAfterHeight int + chainGetTipSet int + chainHead int } func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { tc.chainGetTipSetByHeight++ return &types.TipSet{}, nil } +func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSetAfterHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { + tc.chainGetTipSetAfterHeight++ + return &types.TipSet{}, nil +} func (tc *tsCacheAPIStorageCallCounter) ChainHead(ctx context.Context) (*types.TipSet, error) { tc.chainHead++ return &types.TipSet{}, nil } +func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + tc.chainGetTipSet++ + return &types.TipSet{}, nil +} func TestTsCacheEmpty(t *testing.T) { // Calling best on an empty cache should just call out to the chain API callCounter := &tsCacheAPIStorageCallCounter{t: t} - tsc := newTSCache(50, callCounter) - _, err := tsc.best() + tsc := newTSCache(callCounter, 50) + _, err := tsc.ChainHead(context.Background()) require.NoError(t, err) require.Equal(t, 1, callCounter.chainHead) } + +func TestTsCacheSkip(t *testing.T) { + h := newCacheharness(t) + + ts, err := types.NewTipSet([]*types.BlockHeader{{ + Miner: h.miner, + Height: h.height, + ParentStateRoot: dummyCid, + Messages: dummyCid, + ParentMessageReceipts: dummyCid, + BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, + BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, + // With parents that don't match the last block. + Parents: nil, + }}) + require.NoError(h.t, err) + err = h.tsc.add(ts) + require.Error(t, err) +} diff --git a/chain/events/utils.go b/chain/events/utils.go index 91ea0cd7a..0bfb58e0a 100644 --- a/chain/events/utils.go +++ b/chain/events/utils.go @@ -10,10 +10,10 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -func (me *messageEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd MsgHandler) CheckFunc { +func (me *messageEvents) CheckMsg(smsg types.ChainMsg, hnd MsgHandler) CheckFunc { msg := smsg.VMMessage() - return func(ts *types.TipSet) (done bool, more bool, err error) { + return func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { fa, err := me.cs.StateGetActor(ctx, msg.From, ts.Key()) if err != nil { return false, true, err @@ -24,7 +24,7 @@ func (me *messageEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd return false, true, nil } - ml, err := me.cs.StateSearchMsg(me.ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true) + ml, err := me.cs.StateSearchMsg(ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true) if err != nil { return false, true, xerrors.Errorf("getting receipt in CheckMsg: %w", err) } diff --git a/chain/store/store.go b/chain/store/store.go index df5936c37..1c90b7e0c 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -31,7 +31,6 @@ import ( lru "github.com/hashicorp/golang-lru" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" dstore "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" cbor "github.com/ipfs/go-ipld-cbor" @@ -294,27 +293,36 @@ func (cs *ChainStore) SubHeadChanges(ctx context.Context) chan []*api.HeadChange }} go func() { - defer close(out) - var unsubOnce sync.Once + defer func() { + // Tell the caller we're done first, the following may block for a bit. + close(out) + + // Unsubscribe. + cs.bestTips.Unsub(subch) + + // Drain the channel. + for range subch { + } + }() for { select { case val, ok := <-subch: if !ok { - log.Warn("chain head sub exit loop") + // Shutting down. + return + } + select { + case out <- val.([]*api.HeadChange): + default: + log.Errorf("closing head change subscription due to slow reader") return } if len(out) > 5 { log.Warnf("head change sub is slow, has %d buffered entries", len(out)) } - select { - case out <- val.([]*api.HeadChange): - case <-ctx.Done(): - } case <-ctx.Done(): - unsubOnce.Do(func() { - go cs.bestTips.Unsub(subch) - }) + return } } }() @@ -642,7 +650,7 @@ func (cs *ChainStore) FlushValidationCache() error { return FlushValidationCache(cs.metadataDs) } -func FlushValidationCache(ds datastore.Batching) error { +func FlushValidationCache(ds dstore.Batching) error { log.Infof("clearing block validation cache...") dsWalk, err := ds.Query(query.Query{ @@ -674,7 +682,7 @@ func FlushValidationCache(ds datastore.Batching) error { for _, k := range allKeys { if strings.HasPrefix(k.Key, blockValidationCacheKeyPrefix.String()) { delCnt++ - batch.Delete(datastore.RawKey(k.Key)) // nolint:errcheck + batch.Delete(dstore.RawKey(k.Key)) // nolint:errcheck } } diff --git a/chain/types/message_fuzz.go b/chain/types/message_fuzz.go index 4ef5f6ba2..b57bfe452 100644 --- a/chain/types/message_fuzz.go +++ b/chain/types/message_fuzz.go @@ -1,4 +1,5 @@ -//+build gofuzz +//go:build gofuzz +// +build gofuzz package types diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 31fc738e5..6aa62be51 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -1,3 +1,4 @@ +//go:build !nodaemon // +build !nodaemon package main diff --git a/cmd/lotus/daemon_nodaemon.go b/cmd/lotus/daemon_nodaemon.go index a11d92c6c..7cf12dac4 100644 --- a/cmd/lotus/daemon_nodaemon.go +++ b/cmd/lotus/daemon_nodaemon.go @@ -1,3 +1,4 @@ +//go:build nodaemon // +build nodaemon package main diff --git a/cmd/lotus/debug_advance.go b/cmd/lotus/debug_advance.go index 4e74a995f..8af4f3660 100644 --- a/cmd/lotus/debug_advance.go +++ b/cmd/lotus/debug_advance.go @@ -1,3 +1,4 @@ +//go:build debug // +build debug package main diff --git a/extern/sector-storage/ffiwrapper/prover_cgo.go b/extern/sector-storage/ffiwrapper/prover_cgo.go index 3ad73c81c..1ca22083d 100644 --- a/extern/sector-storage/ffiwrapper/prover_cgo.go +++ b/extern/sector-storage/ffiwrapper/prover_cgo.go @@ -1,4 +1,5 @@ -//+build cgo +//go:build cgo +// +build cgo package ffiwrapper diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index 820c53c4b..59770ec9a 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -1,4 +1,5 @@ -//+build cgo +//go:build cgo +// +build cgo package ffiwrapper diff --git a/extern/sector-storage/ffiwrapper/verifier_cgo.go b/extern/sector-storage/ffiwrapper/verifier_cgo.go index 95724bb7c..ff35ddc1f 100644 --- a/extern/sector-storage/ffiwrapper/verifier_cgo.go +++ b/extern/sector-storage/ffiwrapper/verifier_cgo.go @@ -1,4 +1,5 @@ -//+build cgo +//go:build cgo +// +build cgo package ffiwrapper diff --git a/extern/sector-storage/fsutil/dealloc_other.go b/extern/sector-storage/fsutil/dealloc_other.go index 4f8347951..ff373d97d 100644 --- a/extern/sector-storage/fsutil/dealloc_other.go +++ b/extern/sector-storage/fsutil/dealloc_other.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package fsutil diff --git a/gateway/node.go b/gateway/node.go index 84b616f26..69a9d8731 100644 --- a/gateway/node.go +++ b/gateway/node.go @@ -40,6 +40,7 @@ type TargetAPI interface { ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainNotify(context.Context) (<-chan []*api.HeadChange, error) + ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) @@ -216,6 +217,10 @@ func (gw *Node) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, erro return gw.target.ChainNotify(ctx) } +func (gw *Node) ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) { + return gw.target.ChainGetPath(ctx, from, to) +} + func (gw *Node) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { return gw.target.ChainReadObj(ctx, c) } diff --git a/itests/api_test.go b/itests/api_test.go index 01e006fed..c380a6ed8 100644 --- a/itests/api_test.go +++ b/itests/api_test.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/itests/kit" + logging "github.com/ipfs/go-log/v2" ) func TestAPI(t *testing.T) { @@ -39,6 +40,7 @@ func runAPITest(t *testing.T, opts ...interface{}) { t.Run("testConnectTwo", ts.testConnectTwo) t.Run("testMining", ts.testMining) t.Run("testMiningReal", ts.testMiningReal) + t.Run("testSlowNotify", ts.testSlowNotify) t.Run("testSearchMsg", ts.testSearchMsg) t.Run("testNonGenesisMiner", ts.testNonGenesisMiner) } @@ -169,6 +171,51 @@ func (ts *apiSuite) testMiningReal(t *testing.T) { ts.testMining(t) } +func (ts *apiSuite) testSlowNotify(t *testing.T) { + _ = logging.SetLogLevel("rpc", "ERROR") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + full, miner, _ := kit.EnsembleMinimal(t, ts.opts...) + + // Subscribe a bunch of times to make sure we fill up any RPC buffers. + var newHeadsChans []<-chan []*lapi.HeadChange + for i := 0; i < 100; i++ { + newHeads, err := full.ChainNotify(ctx) + require.NoError(t, err) + newHeadsChans = append(newHeadsChans, newHeads) + } + + initHead := (<-newHeadsChans[0])[0] + baseHeight := initHead.Val.Height() + + bm := kit.NewBlockMiner(t, miner) + bm.MineBlocks(ctx, time.Microsecond) + + full.WaitTillChain(ctx, kit.HeightAtLeast(baseHeight+100)) + + // Make sure they were all closed, draining any buffered events first. + for _, ch := range newHeadsChans { + var ok bool + for ok { + select { + case _, ok = <-ch: + default: + t.Fatal("expected new heads channel to be closed") + } + } + } + + // Make sure we can resubscribe and everything still works. + newHeads, err := full.ChainNotify(ctx) + require.NoError(t, err) + for i := 0; i < 10; i++ { + _, ok := <-newHeads + require.True(t, ok, "notify channel closed") + } +} + func (ts *apiSuite) testNonGenesisMiner(t *testing.T) { ctx := context.Background() diff --git a/itests/paych_api_test.go b/itests/paych_api_test.go index 647db21e0..49c23545b 100644 --- a/itests/paych_api_test.go +++ b/itests/paych_api_test.go @@ -103,10 +103,11 @@ func TestPaymentChannelsAPI(t *testing.T) { creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(paymentCreator))) // wait for the receiver to submit their vouchers - ev := events.NewEvents(ctx, paymentCreator) + ev, err := events.NewEvents(ctx, paymentCreator) + require.NoError(t, err) preds := state.NewStatePredicates(paymentCreator) finished := make(chan struct{}) - err = ev.StateChanged(func(ts *types.TipSet) (done bool, more bool, err error) { + err = ev.StateChanged(func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { act, err := paymentCreator.StateGetActor(ctx, channel, ts.Key()) if err != nil { return false, false, err diff --git a/itests/paych_cli_test.go b/itests/paych_cli_test.go index 17e7bcbf6..a4ad1920b 100644 --- a/itests/paych_cli_test.go +++ b/itests/paych_cli_test.go @@ -381,8 +381,9 @@ func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) { // waitForHeight waits for the node to reach the given chain epoch func waitForHeight(ctx context.Context, t *testing.T, node kit.TestFullNode, height abi.ChainEpoch) { atHeight := make(chan struct{}) - chainEvents := events.NewEvents(ctx, node) - err := chainEvents.ChainAt(func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + chainEvents, err := events.NewEvents(ctx, node) + require.NoError(t, err) + err = chainEvents.ChainAt(ctx, func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { close(atHeight) return nil }, nil, 1, height) diff --git a/lib/ulimit/ulimit_freebsd.go b/lib/ulimit/ulimit_freebsd.go index 7e50436f3..a5ff35707 100644 --- a/lib/ulimit/ulimit_freebsd.go +++ b/lib/ulimit/ulimit_freebsd.go @@ -1,3 +1,4 @@ +//go:build freebsd // +build freebsd package ulimit diff --git a/lib/ulimit/ulimit_test.go b/lib/ulimit/ulimit_test.go index ad27a163a..ecd777e3d 100644 --- a/lib/ulimit/ulimit_test.go +++ b/lib/ulimit/ulimit_test.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package ulimit diff --git a/lib/ulimit/ulimit_unix.go b/lib/ulimit/ulimit_unix.go index a351236dc..ac9f4ca90 100644 --- a/lib/ulimit/ulimit_unix.go +++ b/lib/ulimit/ulimit_unix.go @@ -1,3 +1,4 @@ +//go:build darwin || linux || netbsd || openbsd // +build darwin linux netbsd openbsd package ulimit diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 80ead2be3..2ffa56f5f 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -50,11 +50,14 @@ type clientApi struct { full.MpoolAPI } -func NewClientNodeAdapter(mctx helpers.MetricsCtx, lc fx.Lifecycle, stateapi full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, fundmgr *market.FundManager) storagemarket.StorageClientNode { +func NewClientNodeAdapter(mctx helpers.MetricsCtx, lc fx.Lifecycle, stateapi full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, fundmgr *market.FundManager) (storagemarket.StorageClientNode, error) { capi := &clientApi{chain, stateapi, mpool} ctx := helpers.LifecycleCtx(mctx, lc) - ev := events.NewEvents(ctx, capi) + ev, err := events.NewEvents(ctx, capi) + if err != nil { + return nil, err + } a := &ClientNodeAdapter{ clientApi: capi, @@ -63,7 +66,7 @@ func NewClientNodeAdapter(mctx helpers.MetricsCtx, lc fx.Lifecycle, stateapi ful dsMatcher: newDealStateMatcher(state.NewStatePredicates(state.WrapFastAPI(capi))), } a.scMgr = NewSectorCommittedManager(ev, a, &apiWrapper{api: capi}) - return a + return a, nil } func (c *ClientNodeAdapter) ListStorageProviders(ctx context.Context, encodedTs shared.TipSetToken) ([]*storagemarket.StorageProviderInfo, error) { @@ -262,7 +265,7 @@ func (c *ClientNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID a } // Called immediately to check if the deal has already expired or been slashed - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + checkFunc := func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { if ts == nil { // keep listening for events return false, true, nil diff --git a/markets/storageadapter/ondealsectorcommitted.go b/markets/storageadapter/ondealsectorcommitted.go index 31bc0b8bf..4cd0a2d68 100644 --- a/markets/storageadapter/ondealsectorcommitted.go +++ b/markets/storageadapter/ondealsectorcommitted.go @@ -22,7 +22,7 @@ import ( ) type eventsCalledAPI interface { - Called(check events.CheckFunc, msgHnd events.MsgHandler, rev events.RevertHandler, confidence int, timeout abi.ChainEpoch, mf events.MsgMatchFunc) error + Called(ctx context.Context, check events.CheckFunc, msgHnd events.MsgHandler, rev events.RevertHandler, confidence int, timeout abi.ChainEpoch, mf events.MsgMatchFunc) error } type dealInfoAPI interface { @@ -64,7 +64,7 @@ func (mgr *SectorCommittedManager) OnDealSectorPreCommitted(ctx context.Context, } // First check if the deal is already active, and if so, bail out - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + checkFunc := func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { dealInfo, isActive, err := mgr.checkIfDealAlreadyActive(ctx, ts, &proposal, publishCid) if err != nil { // Note: the error returned from here will end up being returned @@ -165,7 +165,7 @@ func (mgr *SectorCommittedManager) OnDealSectorPreCommitted(ctx context.Context, return nil } - if err := mgr.ev.Called(checkFunc, called, revert, int(build.MessageConfidence+1), timeoutEpoch, matchEvent); err != nil { + if err := mgr.ev.Called(ctx, checkFunc, called, revert, int(build.MessageConfidence+1), timeoutEpoch, matchEvent); err != nil { return xerrors.Errorf("failed to set up called handler: %w", err) } @@ -182,7 +182,7 @@ func (mgr *SectorCommittedManager) OnDealSectorCommitted(ctx context.Context, pr } // First check if the deal is already active, and if so, bail out - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + checkFunc := func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { _, isActive, err := mgr.checkIfDealAlreadyActive(ctx, ts, &proposal, publishCid) if err != nil { // Note: the error returned from here will end up being returned @@ -257,7 +257,7 @@ func (mgr *SectorCommittedManager) OnDealSectorCommitted(ctx context.Context, pr return nil } - if err := mgr.ev.Called(checkFunc, called, revert, int(build.MessageConfidence+1), timeoutEpoch, matchEvent); err != nil { + if err := mgr.ev.Called(ctx, checkFunc, called, revert, int(build.MessageConfidence+1), timeoutEpoch, matchEvent); err != nil { return xerrors.Errorf("failed to set up called handler: %w", err) } diff --git a/markets/storageadapter/ondealsectorcommitted_test.go b/markets/storageadapter/ondealsectorcommitted_test.go index db56ee651..86c01799a 100644 --- a/markets/storageadapter/ondealsectorcommitted_test.go +++ b/markets/storageadapter/ondealsectorcommitted_test.go @@ -477,13 +477,13 @@ type fakeEvents struct { DealStartEpochTimeout bool } -func (fe *fakeEvents) Called(check events.CheckFunc, msgHnd events.MsgHandler, rev events.RevertHandler, confidence int, timeout abi.ChainEpoch, mf events.MsgMatchFunc) error { +func (fe *fakeEvents) Called(ctx context.Context, check events.CheckFunc, msgHnd events.MsgHandler, rev events.RevertHandler, confidence int, timeout abi.ChainEpoch, mf events.MsgMatchFunc) error { if fe.DealStartEpochTimeout { msgHnd(nil, nil, nil, 100) // nolint:errcheck return nil } - _, more, err := check(fe.CheckTs) + _, more, err := check(ctx, fe.CheckTs) if err != nil { return err } @@ -506,7 +506,7 @@ func (fe *fakeEvents) Called(check events.CheckFunc, msgHnd events.MsgHandler, r return nil } if matchMessage.doesRevert { - err := rev(fe.Ctx, matchMessage.ts) + err := rev(ctx, matchMessage.ts) if err != nil { return err } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 23a3c32a8..5c82d0dc8 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -55,11 +55,14 @@ type ProviderNodeAdapter struct { scMgr *SectorCommittedManager } -func NewProviderNodeAdapter(fc *config.MinerFeeConfig, dc *config.DealmakingConfig) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, secb *sectorblocks.SectorBlocks, full v1api.FullNode, dealPublisher *DealPublisher) storagemarket.StorageProviderNode { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, secb *sectorblocks.SectorBlocks, full v1api.FullNode, dealPublisher *DealPublisher) storagemarket.StorageProviderNode { +func NewProviderNodeAdapter(fc *config.MinerFeeConfig, dc *config.DealmakingConfig) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, secb *sectorblocks.SectorBlocks, full v1api.FullNode, dealPublisher *DealPublisher) (storagemarket.StorageProviderNode, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, secb *sectorblocks.SectorBlocks, full v1api.FullNode, dealPublisher *DealPublisher) (storagemarket.StorageProviderNode, error) { ctx := helpers.LifecycleCtx(mctx, lc) - ev := events.NewEvents(ctx, full) + ev, err := events.NewEvents(ctx, full) + if err != nil { + return nil, err + } na := &ProviderNodeAdapter{ FullNode: full, @@ -77,7 +80,7 @@ func NewProviderNodeAdapter(fc *config.MinerFeeConfig, dc *config.DealmakingConf } na.scMgr = NewSectorCommittedManager(ev, na, &apiWrapper{api: full}) - return na + return na, nil } } @@ -340,7 +343,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID } // Called immediately to check if the deal has already expired or been slashed - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + checkFunc := func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { if ts == nil { // keep listening for events return false, true, nil diff --git a/paychmgr/settler/settler.go b/paychmgr/settler/settler.go index ce31ab223..38fcc3b92 100644 --- a/paychmgr/settler/settler.go +++ b/paychmgr/settler/settler.go @@ -56,8 +56,11 @@ func SettlePaymentChannels(mctx helpers.MetricsCtx, lc fx.Lifecycle, papi API) e lc.Append(fx.Hook{ OnStart: func(context.Context) error { pcs := newPaymentChannelSettler(ctx, &papi) - ev := events.NewEvents(ctx, papi) - return ev.Called(pcs.check, pcs.messageHandler, pcs.revertHandler, int(build.MessageConfidence+1), events.NoTimeout, pcs.matcher) + ev, err := events.NewEvents(ctx, &papi) + if err != nil { + return err + } + return ev.Called(ctx, pcs.check, pcs.messageHandler, pcs.revertHandler, int(build.MessageConfidence+1), events.NoTimeout, pcs.matcher) }, }) return nil @@ -70,7 +73,7 @@ func newPaymentChannelSettler(ctx context.Context, api settlerAPI) *paymentChann } } -func (pcs *paymentChannelSettler) check(ts *types.TipSet) (done bool, more bool, err error) { +func (pcs *paymentChannelSettler) check(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) { return false, true, nil } diff --git a/storage/adapter_events.go b/storage/adapter_events.go index ff69c1e51..0f9c62039 100644 --- a/storage/adapter_events.go +++ b/storage/adapter_events.go @@ -21,7 +21,7 @@ func NewEventsAdapter(api *events.Events) EventsAdapter { } func (e EventsAdapter) ChainAt(hnd sealing.HeightHandler, rev sealing.RevertHandler, confidence int, h abi.ChainEpoch) error { - return e.delegate.ChainAt(func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { + return e.delegate.ChainAt(context.TODO(), func(ctx context.Context, ts *types.TipSet, curH abi.ChainEpoch) error { return hnd(ctx, ts.Key().Bytes(), curH) }, func(ctx context.Context, ts *types.TipSet) error { return rev(ctx, ts.Key().Bytes()) diff --git a/storage/miner.go b/storage/miner.go index c4bf41c12..155f5f30d 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -116,6 +116,7 @@ type fullNodeFilteredAPI interface { ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) + ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) ChainHasObj(context.Context, cid.Cid) (bool, error) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) @@ -167,28 +168,29 @@ func (m *Miner) Run(ctx context.Context) error { return xerrors.Errorf("getting miner info: %w", err) } - var ( - // consumer of chain head changes. - evts = events.NewEvents(ctx, m.api) - evtsAdapter = NewEventsAdapter(evts) + // consumer of chain head changes. + evts, err := events.NewEvents(ctx, m.api) + if err != nil { + return xerrors.Errorf("failed to subscribe to events: %w", err) + } + evtsAdapter := NewEventsAdapter(evts) - // Create a shim to glue the API required by the sealing component - // with the API that Lotus is capable of providing. - // The shim translates between "tipset tokens" and tipset keys, and - // provides extra methods. - adaptedAPI = NewSealingAPIAdapter(m.api) + // Create a shim to glue the API required by the sealing component + // with the API that Lotus is capable of providing. + // The shim translates between "tipset tokens" and tipset keys, and + // provides extra methods. + adaptedAPI := NewSealingAPIAdapter(m.api) - // Instantiate a precommit policy. - cfg = sealing.GetSealingConfigFunc(m.getSealConfig) - provingBuffer = md.WPoStProvingPeriod * 2 + // Instantiate a precommit policy. + cfg := sealing.GetSealingConfigFunc(m.getSealConfig) + provingBuffer := md.WPoStProvingPeriod * 2 - pcp = sealing.NewBasicPreCommitPolicy(adaptedAPI, cfg, provingBuffer) + pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, cfg, provingBuffer) - // address selector. - as = func(ctx context.Context, mi miner.MinerInfo, use api.AddrUse, goodFunds, minFunds abi.TokenAmount) (address.Address, abi.TokenAmount, error) { - return m.addrSel.AddressFor(ctx, m.api, mi, use, goodFunds, minFunds) - } - ) + // address selector. + as := func(ctx context.Context, mi miner.MinerInfo, use api.AddrUse, goodFunds, minFunds abi.TokenAmount) (address.Address, abi.TokenAmount, error) { + return m.addrSel.AddressFor(ctx, m.api, mi, use, goodFunds, minFunds) + } // Instantiate the sealing FSM. m.sealing = sealing.New(ctx, adaptedAPI, m.feeCfg, evtsAdapter, m.maddr, m.ds, m.sealer, m.sc, m.verif, m.prover, &pcp, cfg, m.handleSealingNotifications, as) diff --git a/tools/stats/rpc.go b/tools/stats/rpc.go index 0aa3d141e..4e503cb39 100644 --- a/tools/stats/rpc.go +++ b/tools/stats/rpc.go @@ -139,7 +139,10 @@ func GetTips(ctx context.Context, api v0api.FullNode, lastHeight abi.ChainEpoch, for { select { - case changes := <-notif: + case changes, ok := <-notif: + if !ok { + return + } for _, change := range changes { log.Infow("Head event", "height", change.Val.Height(), "type", change.Type)