2019-09-05 07:40:50 +00:00
|
|
|
package events
|
2019-09-03 17:45:55 +00:00
|
|
|
|
|
|
|
import (
|
2019-09-18 11:01:52 +00:00
|
|
|
"context"
|
2019-09-03 17:45:55 +00:00
|
|
|
"fmt"
|
2019-09-05 07:40:50 +00:00
|
|
|
"testing"
|
2019-09-18 11:01:52 +00:00
|
|
|
"time"
|
2019-09-05 07:40:50 +00:00
|
|
|
|
2019-10-18 04:47:41 +00:00
|
|
|
"github.com/filecoin-project/lotus/api"
|
|
|
|
"github.com/filecoin-project/lotus/chain/store"
|
2019-09-27 23:55:15 +00:00
|
|
|
|
2019-09-03 17:45:55 +00:00
|
|
|
"github.com/ipfs/go-cid"
|
2019-09-04 18:56:06 +00:00
|
|
|
"github.com/multiformats/go-multihash"
|
2019-09-03 17:45:55 +00:00
|
|
|
"github.com/stretchr/testify/require"
|
2019-09-05 07:40:50 +00:00
|
|
|
|
2019-10-18 04:47:41 +00:00
|
|
|
"github.com/filecoin-project/lotus/build"
|
|
|
|
"github.com/filecoin-project/lotus/chain/address"
|
|
|
|
"github.com/filecoin-project/lotus/chain/types"
|
2019-09-03 17:45:55 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var dummyCid cid.Cid
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
dummyCid, _ = cid.Parse("bafkqaaa")
|
|
|
|
}
|
|
|
|
|
|
|
|
type fakeMsg struct {
|
|
|
|
bmsgs []*types.Message
|
|
|
|
smsgs []*types.SignedMessage
|
|
|
|
}
|
|
|
|
|
|
|
|
type fakeCS struct {
|
|
|
|
t *testing.T
|
|
|
|
h uint64
|
|
|
|
tsc *tipSetCache
|
|
|
|
|
2019-09-18 11:45:52 +00:00
|
|
|
msgs map[cid.Cid]fakeMsg
|
|
|
|
blkMsgs map[cid.Cid]cid.Cid
|
2019-09-03 17:45:55 +00:00
|
|
|
|
2019-09-18 11:01:52 +00:00
|
|
|
sub func(rev, app []*types.TipSet)
|
2019-09-03 17:45:55 +00:00
|
|
|
}
|
|
|
|
|
2019-11-19 21:27:25 +00:00
|
|
|
func (fcs *fakeCS) StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (fcs *fakeCS) StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) {
|
|
|
|
panic("Not Implemented")
|
|
|
|
}
|
|
|
|
|
2019-09-18 13:32:21 +00:00
|
|
|
func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet) (*types.TipSet, error) {
|
|
|
|
panic("Not Implemented")
|
|
|
|
}
|
|
|
|
|
2019-09-03 17:45:55 +00:00
|
|
|
func makeTs(t *testing.T, h uint64, msgcid cid.Cid) *types.TipSet {
|
2019-10-14 11:53:20 +00:00
|
|
|
a, _ := address.NewFromString("t00")
|
2019-11-08 20:25:02 +00:00
|
|
|
b, _ := address.NewFromString("t02")
|
2019-09-03 17:45:55 +00:00
|
|
|
ts, err := types.NewTipSet([]*types.BlockHeader{
|
|
|
|
{
|
|
|
|
Height: h,
|
2019-10-14 11:53:20 +00:00
|
|
|
Miner: a,
|
2019-09-03 17:45:55 +00:00
|
|
|
|
2019-11-19 15:53:00 +00:00
|
|
|
Ticket: &types.Ticket{[]byte{byte(h % 2)}},
|
2019-11-08 20:25:02 +00:00
|
|
|
|
|
|
|
ParentStateRoot: dummyCid,
|
|
|
|
Messages: msgcid,
|
|
|
|
ParentMessageReceipts: dummyCid,
|
2019-11-19 15:51:12 +00:00
|
|
|
|
2019-11-19 20:07:16 +00:00
|
|
|
BlockSig: &types.Signature{Type: types.KTBLS},
|
2019-11-19 15:51:12 +00:00
|
|
|
BLSAggregate: types.Signature{Type: types.KTBLS},
|
2019-11-08 20:25:02 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
Height: h,
|
|
|
|
Miner: b,
|
|
|
|
|
2019-11-19 15:53:00 +00:00
|
|
|
Ticket: &types.Ticket{[]byte{byte((h + 1) % 2)}},
|
2019-11-08 20:25:02 +00:00
|
|
|
|
2019-09-27 23:55:15 +00:00
|
|
|
ParentStateRoot: dummyCid,
|
|
|
|
Messages: msgcid,
|
|
|
|
ParentMessageReceipts: dummyCid,
|
2019-11-19 15:51:12 +00:00
|
|
|
|
2019-11-19 20:07:16 +00:00
|
|
|
BlockSig: &types.Signature{Type: types.KTBLS},
|
2019-11-19 15:51:12 +00:00
|
|
|
BLSAggregate: types.Signature{Type: types.KTBLS},
|
2019-09-03 17:45:55 +00:00
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
return ts
|
|
|
|
}
|
|
|
|
|
2019-09-18 11:01:52 +00:00
|
|
|
func (fcs *fakeCS) ChainNotify(context.Context) (<-chan []*store.HeadChange, error) {
|
|
|
|
out := make(chan []*store.HeadChange, 1)
|
|
|
|
out <- []*store.HeadChange{{Type: store.HCCurrent, Val: fcs.tsc.best()}}
|
|
|
|
|
|
|
|
fcs.sub = func(rev, app []*types.TipSet) {
|
|
|
|
notif := make([]*store.HeadChange, len(rev)+len(app))
|
|
|
|
|
|
|
|
for i, r := range rev {
|
|
|
|
notif[i] = &store.HeadChange{
|
|
|
|
Type: store.HCRevert,
|
|
|
|
Val: r,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for i, r := range app {
|
|
|
|
notif[i+len(rev)] = &store.HeadChange{
|
|
|
|
Type: store.HCApply,
|
|
|
|
Val: r,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
out <- notif
|
2019-09-03 17:45:55 +00:00
|
|
|
}
|
|
|
|
|
2019-09-18 11:01:52 +00:00
|
|
|
return out, nil
|
2019-09-03 17:45:55 +00:00
|
|
|
}
|
|
|
|
|
2019-09-18 11:45:52 +00:00
|
|
|
func (fcs *fakeCS) ChainGetBlockMessages(ctx context.Context, blk cid.Cid) (*api.BlockMessages, error) {
|
|
|
|
messages, ok := fcs.blkMsgs[blk]
|
|
|
|
if !ok {
|
|
|
|
return &api.BlockMessages{}, nil
|
|
|
|
}
|
|
|
|
|
2019-09-18 11:01:52 +00:00
|
|
|
ms, ok := fcs.msgs[messages]
|
2019-09-18 11:45:52 +00:00
|
|
|
if !ok {
|
|
|
|
return &api.BlockMessages{}, nil
|
2019-09-03 17:45:55 +00:00
|
|
|
}
|
2019-09-18 11:45:52 +00:00
|
|
|
return &api.BlockMessages{BlsMessages: ms.bmsgs, SecpkMessages: ms.smsgs}, nil
|
2019-09-03 17:45:55 +00:00
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func (fcs *fakeCS) fakeMsgs(m fakeMsg) cid.Cid {
|
|
|
|
n := len(fcs.msgs)
|
|
|
|
c, err := cid.Prefix{
|
|
|
|
Version: 1,
|
|
|
|
Codec: cid.Raw,
|
|
|
|
MhType: multihash.IDENTITY,
|
|
|
|
MhLength: -1,
|
|
|
|
}.Sum([]byte(fmt.Sprintf("%d", n)))
|
|
|
|
require.NoError(fcs.t, err)
|
|
|
|
|
|
|
|
fcs.msgs[c] = m
|
|
|
|
return c
|
|
|
|
}
|
|
|
|
|
2019-10-04 22:43:04 +00:00
|
|
|
func (fcs *fakeCS) advance(rev, app int, msgs map[int]cid.Cid, nulls ...int) { // todo: allow msgs
|
2019-09-03 17:45:55 +00:00
|
|
|
if fcs.sub == nil {
|
|
|
|
fcs.t.Fatal("sub not be nil")
|
|
|
|
}
|
|
|
|
|
2019-10-04 22:43:04 +00:00
|
|
|
nullm := map[int]struct{}{}
|
|
|
|
for _, v := range nulls {
|
|
|
|
nullm[v] = struct{}{}
|
|
|
|
}
|
|
|
|
|
2019-09-03 17:45:55 +00:00
|
|
|
var revs []*types.TipSet
|
|
|
|
for i := 0; i < rev; i++ {
|
|
|
|
ts := fcs.tsc.best()
|
|
|
|
|
2019-10-04 22:43:04 +00:00
|
|
|
if _, ok := nullm[int(ts.Height())]; !ok {
|
|
|
|
revs = append(revs, ts)
|
|
|
|
require.NoError(fcs.t, fcs.tsc.revert(ts))
|
|
|
|
}
|
2019-09-03 17:45:55 +00:00
|
|
|
fcs.h--
|
|
|
|
}
|
|
|
|
|
2019-10-04 22:43:04 +00:00
|
|
|
var apps []*types.TipSet
|
2019-09-03 17:45:55 +00:00
|
|
|
for i := 0; i < app; i++ {
|
|
|
|
fcs.h++
|
|
|
|
|
2019-09-18 11:45:52 +00:00
|
|
|
mc, hasMsgs := msgs[i]
|
|
|
|
if !hasMsgs {
|
2019-09-03 17:45:55 +00:00
|
|
|
mc = dummyCid
|
|
|
|
}
|
|
|
|
|
2019-10-04 22:43:04 +00:00
|
|
|
if _, ok := nullm[int(fcs.h)]; ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2019-09-03 17:45:55 +00:00
|
|
|
ts := makeTs(fcs.t, fcs.h, mc)
|
|
|
|
require.NoError(fcs.t, fcs.tsc.add(ts))
|
|
|
|
|
2019-09-18 11:45:52 +00:00
|
|
|
if hasMsgs {
|
|
|
|
fcs.blkMsgs[ts.Blocks()[0].Cid()] = mc
|
|
|
|
}
|
|
|
|
|
2019-10-04 22:43:04 +00:00
|
|
|
apps = append(apps, ts)
|
|
|
|
}
|
|
|
|
|
2019-09-18 11:01:52 +00:00
|
|
|
fcs.sub(revs, apps)
|
|
|
|
time.Sleep(100 * time.Millisecond) // TODO: :c
|
2019-09-03 17:45:55 +00:00
|
|
|
}
|
|
|
|
|
2019-09-18 11:01:52 +00:00
|
|
|
var _ eventApi = &fakeCS{}
|
2019-09-03 17:45:55 +00:00
|
|
|
|
|
|
|
func TestAt(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
2019-09-18 13:32:21 +00:00
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
2019-09-03 17:45:55 +00:00
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
2019-09-18 18:07:39 +00:00
|
|
|
events := NewEvents(context.Background(), fcs)
|
2019-09-03 17:45:55 +00:00
|
|
|
|
|
|
|
var applied bool
|
|
|
|
var reverted bool
|
|
|
|
|
2019-11-05 14:03:59 +00:00
|
|
|
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
2019-09-03 17:45:55 +00:00
|
|
|
require.Equal(t, 5, int(ts.Height()))
|
|
|
|
require.Equal(t, 8, int(curH))
|
|
|
|
applied = true
|
|
|
|
return nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-09-03 17:45:55 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 5)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
fcs.advance(0, 3, nil)
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
fcs.advance(0, 3, nil)
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
fcs.advance(0, 3, nil)
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
applied = false
|
|
|
|
|
|
|
|
fcs.advance(0, 3, nil)
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
2019-10-21 07:54:36 +00:00
|
|
|
fcs.advance(10, 10, nil)
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, true, reverted)
|
|
|
|
applied = false
|
|
|
|
reverted = false
|
|
|
|
|
2019-09-03 17:45:55 +00:00
|
|
|
fcs.advance(10, 1, nil)
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, true, reverted)
|
|
|
|
reverted = false
|
|
|
|
|
|
|
|
fcs.advance(0, 1, nil)
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
fcs.advance(0, 2, nil)
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
fcs.advance(0, 1, nil) // 8
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
2019-09-18 11:10:23 +00:00
|
|
|
}
|
|
|
|
|
2019-10-04 22:43:04 +00:00
|
|
|
func TestAtNullTrigger(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
|
|
|
events := NewEvents(context.Background(), fcs)
|
|
|
|
|
|
|
|
var applied bool
|
|
|
|
var reverted bool
|
|
|
|
|
2019-11-05 14:03:59 +00:00
|
|
|
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
2019-10-05 23:43:10 +00:00
|
|
|
require.Equal(t, uint64(6), ts.Height())
|
2019-10-04 22:43:04 +00:00
|
|
|
require.Equal(t, 8, int(curH))
|
|
|
|
applied = true
|
|
|
|
return nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-10-04 22:43:04 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 5)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
fcs.advance(0, 6, nil, 5)
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
fcs.advance(0, 3, nil)
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
applied = false
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAtNullConf(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
|
|
|
events := NewEvents(context.Background(), fcs)
|
|
|
|
|
|
|
|
var applied bool
|
|
|
|
var reverted bool
|
|
|
|
|
2019-11-05 14:03:59 +00:00
|
|
|
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
2019-10-04 22:43:04 +00:00
|
|
|
require.Equal(t, 5, int(ts.Height()))
|
|
|
|
require.Equal(t, 8, int(curH))
|
|
|
|
applied = true
|
|
|
|
return nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-10-04 22:43:04 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 5)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
fcs.advance(0, 6, nil)
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
fcs.advance(0, 3, nil, 8)
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
applied = false
|
|
|
|
|
|
|
|
fcs.advance(7, 1, nil)
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, true, reverted)
|
|
|
|
reverted = false
|
|
|
|
}
|
|
|
|
|
2019-09-18 11:10:23 +00:00
|
|
|
func TestAtStart(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
2019-09-18 13:32:21 +00:00
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
2019-09-18 11:10:23 +00:00
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
2019-09-18 18:07:39 +00:00
|
|
|
events := NewEvents(context.Background(), fcs)
|
2019-09-18 11:10:23 +00:00
|
|
|
|
|
|
|
fcs.advance(0, 5, nil) // 6
|
|
|
|
|
|
|
|
var applied bool
|
|
|
|
var reverted bool
|
|
|
|
|
2019-11-05 14:03:59 +00:00
|
|
|
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
2019-09-18 11:10:23 +00:00
|
|
|
require.Equal(t, 5, int(ts.Height()))
|
|
|
|
require.Equal(t, 8, int(curH))
|
|
|
|
applied = true
|
|
|
|
return nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-09-18 11:10:23 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 5)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
fcs.advance(0, 5, nil) // 11
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAtStartConfidence(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
2019-09-18 13:32:21 +00:00
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
2019-09-18 11:10:23 +00:00
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
2019-09-18 18:07:39 +00:00
|
|
|
events := NewEvents(context.Background(), fcs)
|
2019-09-18 11:10:23 +00:00
|
|
|
|
|
|
|
fcs.advance(0, 10, nil) // 11
|
|
|
|
|
|
|
|
var applied bool
|
|
|
|
var reverted bool
|
|
|
|
|
2019-11-05 14:03:59 +00:00
|
|
|
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
2019-09-18 11:10:23 +00:00
|
|
|
require.Equal(t, 5, int(ts.Height()))
|
|
|
|
require.Equal(t, 11, int(curH))
|
|
|
|
applied = true
|
|
|
|
return nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-09-18 11:10:23 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 5)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
2019-09-03 17:45:55 +00:00
|
|
|
}
|
|
|
|
|
2019-10-11 23:13:07 +00:00
|
|
|
func TestAtChained(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
|
|
|
events := NewEvents(context.Background(), fcs)
|
|
|
|
|
|
|
|
var applied bool
|
|
|
|
var reverted bool
|
|
|
|
|
2019-11-05 14:03:59 +00:00
|
|
|
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
|
|
|
return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
2019-10-11 23:13:07 +00:00
|
|
|
require.Equal(t, 10, int(ts.Height()))
|
|
|
|
applied = true
|
|
|
|
return nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-10-11 23:13:07 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 10)
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-10-11 23:13:07 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 5)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
fcs.advance(0, 15, nil)
|
|
|
|
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAtChainedConfidence(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
|
|
|
events := NewEvents(context.Background(), fcs)
|
|
|
|
|
|
|
|
fcs.advance(0, 15, nil)
|
|
|
|
|
|
|
|
var applied bool
|
|
|
|
var reverted bool
|
|
|
|
|
2019-11-05 14:03:59 +00:00
|
|
|
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
|
|
|
return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
2019-10-11 23:13:07 +00:00
|
|
|
require.Equal(t, 10, int(ts.Height()))
|
|
|
|
applied = true
|
|
|
|
return nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-10-11 23:13:07 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 10)
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-10-11 23:13:07 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 5)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
}
|
|
|
|
|
2019-10-15 02:20:57 +00:00
|
|
|
func TestAtChainedConfidenceNull(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
|
|
|
events := NewEvents(context.Background(), fcs)
|
|
|
|
|
|
|
|
fcs.advance(0, 15, nil, 5)
|
|
|
|
|
|
|
|
var applied bool
|
|
|
|
var reverted bool
|
|
|
|
|
2019-11-05 14:03:59 +00:00
|
|
|
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
|
2019-10-15 02:20:57 +00:00
|
|
|
applied = true
|
|
|
|
require.Equal(t, 6, int(ts.Height()))
|
|
|
|
return nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-10-15 02:20:57 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
|
|
|
}, 3, 5)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
}
|
|
|
|
|
2019-11-19 15:51:12 +00:00
|
|
|
func matchAddrMethod(to address.Address, m uint64) func(msg *types.Message) (bool, error) {
|
|
|
|
return func(msg *types.Message) (bool, error) {
|
|
|
|
return to == msg.To && m == msg.Method, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-03 17:45:55 +00:00
|
|
|
func TestCalled(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
|
|
|
|
2019-09-18 11:45:52 +00:00
|
|
|
msgs: map[cid.Cid]fakeMsg{},
|
|
|
|
blkMsgs: map[cid.Cid]cid.Cid{},
|
2019-09-18 13:32:21 +00:00
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
2019-09-03 17:45:55 +00:00
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
2019-09-18 18:07:39 +00:00
|
|
|
events := NewEvents(context.Background(), fcs)
|
2019-09-03 17:45:55 +00:00
|
|
|
|
|
|
|
t0123, err := address.NewFromString("t0123")
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
2019-09-04 19:41:34 +00:00
|
|
|
more := true
|
2019-09-03 17:45:55 +00:00
|
|
|
var applied, reverted bool
|
|
|
|
var appliedMsg *types.Message
|
|
|
|
var appliedTs *types.TipSet
|
|
|
|
var appliedH uint64
|
|
|
|
|
2019-09-04 20:22:25 +00:00
|
|
|
err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) {
|
|
|
|
return false, true, nil
|
2019-11-19 21:27:25 +00:00
|
|
|
}, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (bool, error) {
|
2019-11-08 20:25:02 +00:00
|
|
|
require.Equal(t, false, applied)
|
2019-09-03 17:45:55 +00:00
|
|
|
applied = true
|
|
|
|
appliedMsg = msg
|
|
|
|
appliedTs = ts
|
|
|
|
appliedH = curH
|
2019-09-04 19:41:34 +00:00
|
|
|
return more, nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-09-03 17:45:55 +00:00
|
|
|
reverted = true
|
|
|
|
return nil
|
2019-11-19 15:51:12 +00:00
|
|
|
}, 3, 20, matchAddrMethod(t0123, 5))
|
2019-09-03 17:45:55 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
// create few blocks to make sure nothing get's randomly called
|
|
|
|
|
|
|
|
fcs.advance(0, 4, nil) // H=5
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
// create blocks with message (but below confidence threshold)
|
|
|
|
|
|
|
|
fcs.advance(0, 3, map[int]cid.Cid{ // msg at H=6; H=8 (confidence=2)
|
|
|
|
0: fcs.fakeMsgs(fakeMsg{
|
|
|
|
bmsgs: []*types.Message{
|
2019-10-14 11:53:20 +00:00
|
|
|
{To: t0123, From: t0123, Method: 5, Nonce: 1},
|
2019-09-03 17:45:55 +00:00
|
|
|
},
|
|
|
|
}),
|
|
|
|
})
|
|
|
|
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
// create additional block so we are above confidence threshold
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(0, 2, nil) // H=10 (confidence=3, apply)
|
2019-09-03 17:45:55 +00:00
|
|
|
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
applied = false
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
require.Equal(t, uint64(7), appliedTs.Height())
|
|
|
|
require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String())
|
|
|
|
require.Equal(t, uint64(10), appliedH)
|
2019-09-03 17:45:55 +00:00
|
|
|
require.Equal(t, t0123, appliedMsg.To)
|
|
|
|
require.Equal(t, uint64(1), appliedMsg.Nonce)
|
|
|
|
require.Equal(t, uint64(5), appliedMsg.Method)
|
|
|
|
|
|
|
|
// revert some blocks, keep the message
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(3, 1, nil) // H=8 (confidence=1)
|
2019-09-03 17:45:55 +00:00
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
// revert the message
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(2, 1, nil) // H=7, we reverted ts with the msg
|
2019-09-03 17:45:55 +00:00
|
|
|
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, true, reverted)
|
|
|
|
reverted = false
|
|
|
|
|
|
|
|
// send new message on different height
|
|
|
|
|
|
|
|
n2msg := fcs.fakeMsgs(fakeMsg{
|
|
|
|
bmsgs: []*types.Message{
|
2019-10-14 11:53:20 +00:00
|
|
|
{To: t0123, From: t0123, Method: 5, Nonce: 2},
|
2019-09-03 17:45:55 +00:00
|
|
|
},
|
|
|
|
})
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(0, 5, map[int]cid.Cid{ // (confidence=3)
|
2019-09-03 17:45:55 +00:00
|
|
|
0: n2msg,
|
|
|
|
})
|
|
|
|
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
applied = false
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
require.Equal(t, uint64(9), appliedTs.Height())
|
|
|
|
require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String())
|
|
|
|
require.Equal(t, uint64(12), appliedH)
|
2019-09-03 17:45:55 +00:00
|
|
|
require.Equal(t, t0123, appliedMsg.To)
|
|
|
|
require.Equal(t, uint64(2), appliedMsg.Nonce)
|
|
|
|
require.Equal(t, uint64(5), appliedMsg.Method)
|
|
|
|
|
|
|
|
// revert and apply at different height
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(4, 6, map[int]cid.Cid{ // (confidence=3)
|
2019-09-03 17:45:55 +00:00
|
|
|
1: n2msg,
|
|
|
|
})
|
|
|
|
|
|
|
|
// TODO: We probably don't want to call revert/apply, as restarting certain
|
|
|
|
// actions may be expensive, and in this case the message is still
|
|
|
|
// on-chain, just at different height
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, true, reverted)
|
|
|
|
reverted = false
|
|
|
|
applied = false
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
require.Equal(t, uint64(11), appliedTs.Height())
|
|
|
|
require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String())
|
|
|
|
require.Equal(t, uint64(14), appliedH)
|
2019-09-03 17:45:55 +00:00
|
|
|
require.Equal(t, t0123, appliedMsg.To)
|
|
|
|
require.Equal(t, uint64(2), appliedMsg.Nonce)
|
|
|
|
require.Equal(t, uint64(5), appliedMsg.Method)
|
2019-09-03 17:59:32 +00:00
|
|
|
|
|
|
|
// call method again
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(0, 5, map[int]cid.Cid{
|
2019-09-03 17:59:32 +00:00
|
|
|
0: n2msg,
|
|
|
|
})
|
|
|
|
|
2019-09-04 18:56:06 +00:00
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
applied = false
|
|
|
|
|
|
|
|
// send and revert below confidence, then cross confidence
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(0, 2, map[int]cid.Cid{
|
2019-09-04 18:56:06 +00:00
|
|
|
0: fcs.fakeMsgs(fakeMsg{
|
|
|
|
bmsgs: []*types.Message{
|
2019-10-14 11:53:20 +00:00
|
|
|
{To: t0123, From: t0123, Method: 5, Nonce: 3},
|
2019-09-04 18:56:06 +00:00
|
|
|
},
|
|
|
|
}),
|
|
|
|
})
|
|
|
|
|
|
|
|
fcs.advance(1, 4, nil) // H=19, but message reverted
|
|
|
|
|
2019-09-04 16:09:08 +00:00
|
|
|
require.Equal(t, false, applied)
|
2019-09-03 17:59:32 +00:00
|
|
|
require.Equal(t, false, reverted)
|
2019-09-04 18:56:06 +00:00
|
|
|
|
2019-09-04 19:41:34 +00:00
|
|
|
// test timeout (it's set to 20 in the call to `events.Called` above)
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(0, 6, nil)
|
2019-09-04 19:41:34 +00:00
|
|
|
|
|
|
|
require.Equal(t, false, applied) // not calling timeout as we received messages
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
// test unregistering with more
|
|
|
|
|
|
|
|
more = false
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(0, 5, map[int]cid.Cid{
|
2019-09-04 19:41:34 +00:00
|
|
|
0: fcs.fakeMsgs(fakeMsg{
|
|
|
|
bmsgs: []*types.Message{
|
2019-10-14 11:53:20 +00:00
|
|
|
{To: t0123, From: t0123, Method: 5, Nonce: 4}, // this signals we don't want more
|
2019-09-04 19:41:34 +00:00
|
|
|
},
|
|
|
|
}),
|
|
|
|
})
|
|
|
|
|
|
|
|
require.Equal(t, true, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
applied = false
|
|
|
|
|
2019-11-07 09:02:29 +00:00
|
|
|
fcs.advance(0, 5, map[int]cid.Cid{
|
2019-09-04 19:41:34 +00:00
|
|
|
0: fcs.fakeMsgs(fakeMsg{
|
|
|
|
bmsgs: []*types.Message{
|
2019-10-14 11:53:20 +00:00
|
|
|
{To: t0123, From: t0123, Method: 5, Nonce: 5},
|
2019-09-04 19:41:34 +00:00
|
|
|
},
|
|
|
|
}),
|
|
|
|
})
|
|
|
|
|
|
|
|
require.Equal(t, false, applied) // should not get any further notifications
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
// revert after disabled
|
|
|
|
|
|
|
|
fcs.advance(5, 1, nil) // try reverting msg sent after disabling
|
|
|
|
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, false, reverted)
|
|
|
|
|
|
|
|
fcs.advance(5, 1, nil) // try reverting msg sent before disabling
|
|
|
|
|
|
|
|
require.Equal(t, false, applied)
|
|
|
|
require.Equal(t, true, reverted)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestCalledTimeout(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
|
|
|
|
2019-09-18 11:45:52 +00:00
|
|
|
msgs: map[cid.Cid]fakeMsg{},
|
|
|
|
blkMsgs: map[cid.Cid]cid.Cid{},
|
2019-09-18 13:32:21 +00:00
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
2019-09-04 19:41:34 +00:00
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
2019-09-18 18:07:39 +00:00
|
|
|
events := NewEvents(context.Background(), fcs)
|
2019-09-04 19:41:34 +00:00
|
|
|
|
|
|
|
t0123, err := address.NewFromString("t0123")
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
called := false
|
|
|
|
|
2019-09-04 20:22:25 +00:00
|
|
|
err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) {
|
|
|
|
return false, true, nil
|
2019-11-19 21:27:25 +00:00
|
|
|
}, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (bool, error) {
|
2019-09-04 19:41:34 +00:00
|
|
|
called = true
|
|
|
|
require.Nil(t, msg)
|
|
|
|
require.Equal(t, uint64(20), ts.Height())
|
|
|
|
require.Equal(t, uint64(23), curH)
|
|
|
|
return false, nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-09-04 19:41:34 +00:00
|
|
|
t.Fatal("revert on timeout")
|
|
|
|
return nil
|
2019-11-19 15:51:12 +00:00
|
|
|
}, 3, 20, matchAddrMethod(t0123, 5))
|
2019-09-04 19:41:34 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
fcs.advance(0, 21, nil)
|
|
|
|
require.False(t, called)
|
|
|
|
|
|
|
|
fcs.advance(0, 5, nil)
|
|
|
|
require.True(t, called)
|
2019-09-04 20:22:25 +00:00
|
|
|
called = false
|
|
|
|
|
|
|
|
// with check func reporting done
|
|
|
|
|
|
|
|
fcs = &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
|
|
|
|
2019-09-18 11:45:52 +00:00
|
|
|
msgs: map[cid.Cid]fakeMsg{},
|
|
|
|
blkMsgs: map[cid.Cid]cid.Cid{},
|
2019-09-18 13:32:21 +00:00
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
2019-09-04 20:22:25 +00:00
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
2019-09-18 18:07:39 +00:00
|
|
|
events = NewEvents(context.Background(), fcs)
|
2019-09-04 20:22:25 +00:00
|
|
|
|
|
|
|
err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) {
|
|
|
|
return true, true, nil
|
2019-11-19 21:27:25 +00:00
|
|
|
}, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (bool, error) {
|
2019-09-04 20:22:25 +00:00
|
|
|
called = true
|
|
|
|
require.Nil(t, msg)
|
|
|
|
require.Equal(t, uint64(20), ts.Height())
|
|
|
|
require.Equal(t, uint64(23), curH)
|
|
|
|
return false, nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-09-04 20:22:25 +00:00
|
|
|
t.Fatal("revert on timeout")
|
|
|
|
return nil
|
2019-11-19 15:51:12 +00:00
|
|
|
}, 3, 20, matchAddrMethod(t0123, 5))
|
2019-09-04 20:22:25 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
fcs.advance(0, 21, nil)
|
|
|
|
require.False(t, called)
|
|
|
|
|
|
|
|
fcs.advance(0, 5, nil)
|
|
|
|
require.False(t, called)
|
2019-09-03 17:45:55 +00:00
|
|
|
}
|
2019-09-05 07:57:12 +00:00
|
|
|
|
|
|
|
func TestCalledOrder(t *testing.T) {
|
|
|
|
fcs := &fakeCS{
|
|
|
|
t: t,
|
|
|
|
h: 1,
|
|
|
|
|
2019-09-18 11:45:52 +00:00
|
|
|
msgs: map[cid.Cid]fakeMsg{},
|
|
|
|
blkMsgs: map[cid.Cid]cid.Cid{},
|
2019-09-18 13:32:21 +00:00
|
|
|
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
|
2019-09-05 07:57:12 +00:00
|
|
|
}
|
|
|
|
require.NoError(t, fcs.tsc.add(makeTs(t, 1, dummyCid)))
|
|
|
|
|
2019-09-18 18:07:39 +00:00
|
|
|
events := NewEvents(context.Background(), fcs)
|
2019-09-05 07:57:12 +00:00
|
|
|
|
|
|
|
t0123, err := address.NewFromString("t0123")
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
at := 0
|
|
|
|
|
|
|
|
err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) {
|
|
|
|
return false, true, nil
|
2019-11-19 21:27:25 +00:00
|
|
|
}, func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (bool, error) {
|
2019-09-05 07:57:12 +00:00
|
|
|
switch at {
|
|
|
|
case 0:
|
|
|
|
require.Equal(t, uint64(1), msg.Nonce)
|
2019-11-07 09:02:29 +00:00
|
|
|
require.Equal(t, uint64(4), ts.Height())
|
2019-09-05 07:57:12 +00:00
|
|
|
case 1:
|
|
|
|
require.Equal(t, uint64(2), msg.Nonce)
|
2019-11-07 09:02:29 +00:00
|
|
|
require.Equal(t, uint64(5), ts.Height())
|
2019-09-05 07:57:12 +00:00
|
|
|
default:
|
|
|
|
t.Fatal("apply should only get called twice, at: ", at)
|
|
|
|
}
|
|
|
|
at++
|
|
|
|
return true, nil
|
2019-11-05 14:03:59 +00:00
|
|
|
}, func(_ context.Context, ts *types.TipSet) error {
|
2019-09-05 07:57:12 +00:00
|
|
|
switch at {
|
|
|
|
case 2:
|
2019-11-07 09:02:29 +00:00
|
|
|
require.Equal(t, uint64(5), ts.Height())
|
2019-09-05 07:57:12 +00:00
|
|
|
case 3:
|
2019-11-07 09:02:29 +00:00
|
|
|
require.Equal(t, uint64(4), ts.Height())
|
2019-09-05 07:57:12 +00:00
|
|
|
default:
|
|
|
|
t.Fatal("revert should only get called twice, at: ", at)
|
|
|
|
}
|
|
|
|
at++
|
|
|
|
return nil
|
2019-11-19 15:51:12 +00:00
|
|
|
}, 3, 20, matchAddrMethod(t0123, 5))
|
2019-09-05 07:57:12 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
fcs.advance(0, 10, map[int]cid.Cid{
|
|
|
|
1: fcs.fakeMsgs(fakeMsg{
|
|
|
|
bmsgs: []*types.Message{
|
2019-10-14 11:53:20 +00:00
|
|
|
{To: t0123, From: t0123, Method: 5, Nonce: 1},
|
2019-09-05 07:57:12 +00:00
|
|
|
},
|
|
|
|
}),
|
|
|
|
2: fcs.fakeMsgs(fakeMsg{
|
|
|
|
bmsgs: []*types.Message{
|
2019-10-14 11:53:20 +00:00
|
|
|
{To: t0123, From: t0123, Method: 5, Nonce: 2},
|
2019-09-05 07:57:12 +00:00
|
|
|
},
|
|
|
|
}),
|
|
|
|
})
|
|
|
|
|
|
|
|
fcs.advance(9, 1, nil)
|
|
|
|
}
|