feat: chain: make fetching tipset by height 1000x faster

chain/store/index.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"os"
 	"strconv"
+	"sync"
 
-	lru "github.com/hashicorp/golang-lru"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
@@ -13,7 +13,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-var DefaultChainIndexCacheSize = 32 << 10
+var DefaultChainIndexCacheSize = 32 << 15
 
 func init() {
 	if s := os.Getenv("LOTUS_CHAIN_INDEX_CACHE"); s != "" {
@@ -27,7 +27,8 @@ func init() {
 }
 
 type ChainIndex struct {
-	skipCache *lru.ARCCache
+	indexCacheLk sync.Mutex
+	indexCache   map[types.TipSetKey]*lbEntry
 
 	loadTipSet loadTipSetFunc
 
@@ -36,17 +37,14 @@ type ChainIndex struct {
 type loadTipSetFunc func(context.Context, types.TipSetKey) (*types.TipSet, error)
 
 func NewChainIndex(lts loadTipSetFunc) *ChainIndex {
-	sc, _ := lru.NewARC(DefaultChainIndexCacheSize)
 	return &ChainIndex{
-		skipCache:  sc,
+		indexCache: make(map[types.TipSetKey]*lbEntry, DefaultChainIndexCacheSize),
 		loadTipSet: lts,
 		skipLength: 20,
 	}
 }
 
 type lbEntry struct {
-	ts           *types.TipSet
-	parentHeight abi.ChainEpoch
 	targetHeight abi.ChainEpoch
 	target       types.TipSetKey
 }
@@ -58,25 +56,36 @@ func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet,
 
 	rounded, err := ci.roundDown(ctx, from)
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("failed to round down: %w", err)
 	}
 
+	ci.indexCacheLk.Lock()
+	defer ci.indexCacheLk.Unlock()
 	cur := rounded.Key()
 	for {
-		cval, ok := ci.skipCache.Get(cur)
+		lbe, ok := ci.indexCache[cur]
 		if !ok {
 			fc, err := ci.fillCache(ctx, cur)
 			if err != nil {
-				return nil, err
+				return nil, xerrors.Errorf("failed to fill cache: %w", err)
 			}
-			cval = fc
+			lbe = fc
 		}
 
-		lbe := cval.(*lbEntry)
-		if lbe.ts.Height() == to || lbe.parentHeight < to {
-			return lbe.ts, nil
-		} else if to > lbe.targetHeight {
-			return ci.walkBack(ctx, lbe.ts, to)
+		if to == lbe.targetHeight {
+			ts, err := ci.loadTipSet(ctx, lbe.target)
+			if err != nil {
+				return nil, xerrors.Errorf("failed to load tipset: %w", err)
+			}
+
+			return ts, nil
+		}
+		if to > lbe.targetHeight {
+			ts, err := ci.loadTipSet(ctx, cur)
+			if err != nil {
+				return nil, xerrors.Errorf("failed to load tipset: %w", err)
+			}
+			return ci.walkBack(ctx, ts, to)
 		}
 
 		cur = lbe.target
@@ -87,16 +96,17 @@ func (ci *ChainIndex) GetTipsetByHeightWithoutCache(ctx context.Context, from *t
 	return ci.walkBack(ctx, from, to)
 }
 
+// Caller must hold indexCacheLk
 func (ci *ChainIndex) fillCache(ctx context.Context, tsk types.TipSetKey) (*lbEntry, error) {
 	ts, err := ci.loadTipSet(ctx, tsk)
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("failed to load tipset: %w", err)
 	}
 
 	if ts.Height() == 0 {
 		return &lbEntry{
-			ts:           ts,
-			parentHeight: 0,
+			targetHeight: 0,
+			target:       tsk,
 		}, nil
 	}
 
@@ -124,12 +134,10 @@ func (ci *ChainIndex) fillCache(ctx context.Context, tsk types.TipSetKey) (*lbEn
 	}
 
 	lbe := &lbEntry{
-		ts:           ts,
-		parentHeight: parent.Height(),
 		targetHeight: skipTarget.Height(),
 		target:       skipTarget.Key(),
 	}
-	ci.skipCache.Add(tsk, lbe)
+	ci.indexCache[tsk] = lbe
 
 	return lbe, nil
 }
@@ -144,7 +152,7 @@ func (ci *ChainIndex) roundDown(ctx context.Context, ts *types.TipSet) (*types.T
 
 	rounded, err := ci.walkBack(ctx, ts, target)
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("failed to walk back: %w", err)
 	}
 
 	return rounded, nil
@@ -164,7 +172,7 @@ func (ci *ChainIndex) walkBack(ctx context.Context, from *types.TipSet, to abi.C
 	for {
 		pts, err := ci.loadTipSet(ctx, ts.Parents())
 		if err != nil {
-			return nil, err
+			return nil, xerrors.Errorf("failed to load tipset: %w", err)
 		}
 
 		if to > pts.Height() {