From 17505a0022c256814c035aab57dcd791158b49e9 Mon Sep 17 00:00:00 2001
From: Fridrik Asmundsson <fridrik01@gmail.com>
Date: Wed, 29 Mar 2023 11:38:26 +0200
Subject: [PATCH] Make execution trace configurable via env variable

We want to make the execution trace cache size configurable as SPs
may want to disable it while exchanges may want to crank it up.

We were also are going with intuition for this value, so having
ability to change it without a new build would help.

Fixes: https://github.com/filecoin-project/lotus/issues/10584
---
 chain/stmgr/execute.go | 26 +++++++++++++++-----------
 chain/stmgr/stmgr.go   | 27 ++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/chain/stmgr/execute.go b/chain/stmgr/execute.go
index 8c85a1031..09fdb6fd0 100644
--- a/chain/stmgr/execute.go
+++ b/chain/stmgr/execute.go
@@ -131,15 +131,17 @@ func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (c
 	tsKey := ts.Key()
 
 	// check if we have the trace for this tipset in the cache
-	sm.execTraceCacheLock.Lock()
-	if entry, ok := sm.execTraceCache.Get(tsKey); ok {
-		// we have to make a deep copy since caller can modify the invocTrace
-		// and we don't want that to change what we store in cache
-		invocTraceCopy := makeDeepCopy(entry.invocTrace)
+	if defaultExecTraceCacheSize > 0 {
+		sm.execTraceCacheLock.Lock()
+		if entry, ok := sm.execTraceCache.Get(tsKey); ok {
+			// we have to make a deep copy since caller can modify the invocTrace
+			// and we don't want that to change what we store in cache
+			invocTraceCopy := makeDeepCopy(entry.invocTrace)
+			sm.execTraceCacheLock.Unlock()
+			return entry.postStateRoot, invocTraceCopy, nil
+		}
 		sm.execTraceCacheLock.Unlock()
-		return entry.postStateRoot, invocTraceCopy, nil
 	}
-	sm.execTraceCacheLock.Unlock()
 
 	var invocTrace []*api.InvocResult
 	st, err := sm.ExecutionTraceWithMonitor(ctx, ts, &InvocationTracer{trace: &invocTrace})
@@ -147,11 +149,13 @@ func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (c
 		return cid.Undef, nil, err
 	}
 
-	invocTraceCopy := makeDeepCopy(invocTrace)
+	if defaultExecTraceCacheSize > 0 {
+		invocTraceCopy := makeDeepCopy(invocTrace)
 
-	sm.execTraceCacheLock.Lock()
-	sm.execTraceCache.Add(tsKey, tipSetCacheEntry{st, invocTraceCopy})
-	sm.execTraceCacheLock.Unlock()
+		sm.execTraceCacheLock.Lock()
+		sm.execTraceCache.Add(tsKey, tipSetCacheEntry{st, invocTraceCopy})
+		sm.execTraceCacheLock.Unlock()
+	}
 
 	return st, invocTrace, nil
 }
diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go
index bf10665e7..aaebfcbe8 100644
--- a/chain/stmgr/stmgr.go
+++ b/chain/stmgr/stmgr.go
@@ -3,6 +3,8 @@ package stmgr
 import (
 	"context"
 	"fmt"
+	"os"
+	"strconv"
 	"sync"
 
 	lru "github.com/hashicorp/golang-lru/v2"
@@ -40,8 +42,7 @@ import (
 const LookbackNoLimit = api.LookbackNoLimit
 const ReceiptAmtBitwidth = 3
 
-const execTraceCacheSize = 16
-
+var defaultExecTraceCacheSize = 16
 var log = logging.Logger("statemgr")
 
 type StateManagerAPI interface {
@@ -74,6 +75,17 @@ func (m *migrationResultCache) keyForMigration(root cid.Cid) dstore.Key {
 	return dstore.NewKey(kStr)
 }
 
+func init() {
+	if s := os.Getenv("LOTUS_EXEC_TRACE_CACHE"); s != "" {
+		letc, err := strconv.Atoi(s)
+		if err != nil {
+			log.Errorf("failed to parse 'LOTUS_EXEC_TRACE_CACHE' env var: %s", err)
+		}
+
+		defaultExecTraceCacheSize = letc
+	}
+}
+
 func (m *migrationResultCache) Get(ctx context.Context, root cid.Cid) (cid.Cid, bool, error) {
 	k := m.keyForMigration(root)
 
@@ -200,9 +212,14 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder,
 		}
 	}
 
-	execTraceCache, err := lru.NewARC[types.TipSetKey, tipSetCacheEntry](execTraceCacheSize)
-	if err != nil {
-		return nil, err
+	log.Debugf("execTraceCache size: %d", defaultExecTraceCacheSize)
+	var execTraceCache *lru.ARCCache[types.TipSetKey, tipSetCacheEntry]
+	var err error
+	if defaultExecTraceCacheSize > 0 {
+		execTraceCache, err = lru.NewARC[types.TipSetKey, tipSetCacheEntry](defaultExecTraceCacheSize)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return &StateManager{