From c97720e731b6c275e41e17c179576c57ba95c451 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 26 Jun 2020 20:42:29 +0200
Subject: [PATCH 01/47] Fix import analyze

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index ebc62aa5d..71457a708 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -253,12 +253,16 @@ var importAnalyzeCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
+		dec := json.NewDecoder(fi)
 
 		var results []TipSetExec
 		for {
 			var tse TipSetExec
-			if err := json.NewDecoder(fi).Decode(&tse); err != nil {
+			if err := dec.Decode(&tse); err != nil {
 				if err != io.EOF {
+					if e, ok := err.(*json.SyntaxError); ok {
+						log.Warnf("syntax error at byte offset %d", e.Offset)
+					}
 					return err
 				}
 				break
@@ -310,6 +314,9 @@ var importAnalyzeCmd = &cli.Command{
 		fmt.Println("Average time per epoch: ", totalTime/time.Duration(len(results)))
 
 		n := 30
+		if len(invocs) < n {
+			n = len(invocs)
+		}
 		fmt.Printf("Top %d most expensive calls:\n", n)
 		for i := 0; i < n; i++ {
 			inv := invocs[i].Invoc

From e65215f1b54cf45d77f2449a062b8bb8099ca0e0 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 26 Jun 2020 21:54:20 +0200
Subject: [PATCH 02/47] Make lotus-bench import analyze multithreaded and less
 RAM hungry

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 89 ++++++++++++++++++++++++++++++---------
 1 file changed, 68 insertions(+), 21 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 71457a708..58de65fb8 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -255,7 +255,55 @@ var importAnalyzeCmd = &cli.Command{
 		}
 		dec := json.NewDecoder(fi)
 
-		var results []TipSetExec
+		const nWorkers = 16
+		tseIn := make(chan TipSetExec, 2*nWorkers)
+		type result struct {
+			totalTime       time.Duration
+			chargeDeltas    map[string][]float64
+			expensiveInvocs []Invocation
+		}
+
+		results := make(chan result, nWorkers)
+
+		for i := 0; i < nWorkers; i++ {
+			go func() {
+				chargeDeltas := make(map[string][]float64)
+				var totalTime time.Duration
+				var expensiveInvocs []Invocation
+				var leastExpensiveInvoc = time.Duration(0)
+
+				for {
+					tse, ok := <-tseIn
+					if !ok {
+						results <- result{
+							totalTime:       totalTime,
+							chargeDeltas:    chargeDeltas,
+							expensiveInvocs: expensiveInvocs,
+						}
+						return
+					}
+					totalTime += tse.Duration
+					for _, inv := range tse.Trace {
+						if inv.Duration > leastExpensiveInvoc {
+							expensiveInvocs = append(expensiveInvocs, Invocation{
+								TipSet: tse.TipSet,
+								Invoc:  inv,
+							})
+						}
+
+						tallyGasCharges(chargeDeltas, &inv.ExecutionTrace)
+					}
+					sort.Slice(expensiveInvocs, func(i, j int) bool {
+						return expensiveInvocs[i].Invoc.Duration > expensiveInvocs[j].Invoc.Duration
+					})
+					if len(expensiveInvocs) != 0 {
+						leastExpensiveInvoc = expensiveInvocs[len(expensiveInvocs)-1].Invoc.Duration
+					}
+				}
+			}()
+		}
+
+		var totalTipsets int64
 		for {
 			var tse TipSetExec
 			if err := dec.Decode(&tse); err != nil {
@@ -267,32 +315,31 @@ var importAnalyzeCmd = &cli.Command{
 				}
 				break
 			}
-			results = append(results, tse)
+			totalTipsets++
+			tseIn <- tse
+			if totalTipsets%10 == 0 {
+				fmt.Printf("\rProcessed %d tipsets", totalTipsets)
+			}
 		}
-
-		chargeDeltas := make(map[string][]float64)
+		close(tseIn)
+		fmt.Printf("\n")
+		fmt.Printf("Collecting results\n")
 
 		var invocs []Invocation
 		var totalTime time.Duration
-		for i, r := range results {
-			_ = i
-			totalTime += r.Duration
-
-			for _, inv := range r.Trace {
-				invocs = append(invocs, Invocation{
-					TipSet: r.TipSet,
-					Invoc:  inv,
-				})
-
-				cgas, vgas := countGasCosts(&inv.ExecutionTrace)
-				fmt.Printf("Invocation: %d %s: %s %d -> %0.2f\n", inv.Msg.Method, inv.Msg.To, inv.Duration, cgas+vgas, float64(GasPerNs*inv.Duration.Nanoseconds())/float64(cgas+vgas))
-
-				tallyGasCharges(chargeDeltas, &inv.ExecutionTrace)
-
+		var keys []string
+		var chargeDeltas = make(map[string][]float64)
+		for i := 0; i < nWorkers; i++ {
+			fmt.Printf("\rProcessing results from worker %d/%d", i+1, nWorkers)
+			res := <-results
+			invocs = append(invocs, res.expensiveInvocs...)
+			for k, v := range res.chargeDeltas {
+				chargeDeltas[k] = append(chargeDeltas[k], v...)
 			}
+			totalTime += res.totalTime
 		}
 
-		var keys []string
+		fmt.Printf("\nCollecting gas keys\n")
 		for k := range chargeDeltas {
 			keys = append(keys, k)
 		}
@@ -311,7 +358,7 @@ var importAnalyzeCmd = &cli.Command{
 		})
 
 		fmt.Println("Total time: ", totalTime)
-		fmt.Println("Average time per epoch: ", totalTime/time.Duration(len(results)))
+		fmt.Println("Average time per epoch: ", totalTime/time.Duration(totalTipsets))
 
 		n := 30
 		if len(invocs) < n {

From db5929f4a54f18b8573558bd0072853076910269 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 26 Jun 2020 21:56:34 +0200
Subject: [PATCH 03/47] Limit size of expensiveInvocs

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 58de65fb8..959629d6a 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -299,6 +299,7 @@ var importAnalyzeCmd = &cli.Command{
 					if len(expensiveInvocs) != 0 {
 						leastExpensiveInvoc = expensiveInvocs[len(expensiveInvocs)-1].Invoc.Duration
 					}
+					expensiveInvocs = expensiveInvocs[:30]
 				}
 			}()
 		}

From 04edeccbf4b74b64b28519f3f7fd8163f41a3369 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 26 Jun 2020 21:57:57 +0200
Subject: [PATCH 04/47] fix trimming

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 959629d6a..8c420e3c9 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -298,8 +298,12 @@ var importAnalyzeCmd = &cli.Command{
 					})
 					if len(expensiveInvocs) != 0 {
 						leastExpensiveInvoc = expensiveInvocs[len(expensiveInvocs)-1].Invoc.Duration
+						n := 30
+						if len(expensiveInvocs) < n {
+							n = len(expensiveInvocs)
+						}
+						expensiveInvocs = expensiveInvocs[:n]
 					}
-					expensiveInvocs = expensiveInvocs[:30]
 				}
 			}()
 		}

From 710286d6e7f827f8e70c58d065005ab77a4d39ba Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 26 Jun 2020 22:18:43 +0200
Subject: [PATCH 05/47] Add pprof

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 8c420e3c9..98a408556 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -7,6 +7,8 @@ import (
 	"io"
 	"io/ioutil"
 	"math"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 	"runtime"
 	"runtime/pprof"
@@ -249,6 +251,10 @@ var importAnalyzeCmd = &cli.Command{
 			return nil
 		}
 
+		go func() {
+			http.ListenAndServe("localhost:6060", nil)
+		}()
+
 		fi, err := os.Open(cctx.Args().First())
 		if err != nil {
 			return err
@@ -282,6 +288,7 @@ var importAnalyzeCmd = &cli.Command{
 						}
 						return
 					}
+
 					totalTime += tse.Duration
 					for _, inv := range tse.Trace {
 						if inv.Duration > leastExpensiveInvoc {

From 87d668390ec1cc529dce3c625a4bc45c8591335f Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 26 Jun 2020 22:26:03 +0200
Subject: [PATCH 06/47] Use float32

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 98a408556..a6e8de3a4 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -210,29 +210,29 @@ func countGasCosts(et *types.ExecutionTrace) (int64, int64) {
 	return cgas, vgas
 }
 
-func compStats(vals []float64) (float64, float64) {
-	var sum float64
+func compStats(vals []float32) (float32, float32) {
+	var sum float32
 
 	for _, v := range vals {
 		sum += v
 	}
 
-	av := sum / float64(len(vals))
+	av := sum / float32(len(vals))
 
-	var varsum float64
+	var varsum float32
 	for _, v := range vals {
 		delta := av - v
 		varsum += delta * delta
 	}
 
-	return av, math.Sqrt(varsum / float64(len(vals)))
+	return av, float32(math.Sqrt(float64(varsum / float32(len(vals)))))
 }
 
-func tallyGasCharges(charges map[string][]float64, et *types.ExecutionTrace) {
+func tallyGasCharges(charges map[string][]float32, et *types.ExecutionTrace) {
 	for _, gc := range et.GasCharges {
 
 		compGas := gc.ComputeGas + gc.VirtualComputeGas
-		ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
+		ratio := float32(compGas) / float32(gc.TimeTaken.Nanoseconds())
 
 		charges[gc.Name] = append(charges[gc.Name], 1/(ratio/GasPerNs))
 		//fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs))
@@ -265,7 +265,7 @@ var importAnalyzeCmd = &cli.Command{
 		tseIn := make(chan TipSetExec, 2*nWorkers)
 		type result struct {
 			totalTime       time.Duration
-			chargeDeltas    map[string][]float64
+			chargeDeltas    map[string][]float32
 			expensiveInvocs []Invocation
 		}
 
@@ -273,7 +273,7 @@ var importAnalyzeCmd = &cli.Command{
 
 		for i := 0; i < nWorkers; i++ {
 			go func() {
-				chargeDeltas := make(map[string][]float64)
+				chargeDeltas := make(map[string][]float32)
 				var totalTime time.Duration
 				var expensiveInvocs []Invocation
 				var leastExpensiveInvoc = time.Duration(0)
@@ -340,7 +340,7 @@ var importAnalyzeCmd = &cli.Command{
 		var invocs []Invocation
 		var totalTime time.Duration
 		var keys []string
-		var chargeDeltas = make(map[string][]float64)
+		var chargeDeltas = make(map[string][]float32)
 		for i := 0; i < nWorkers; i++ {
 			fmt.Printf("\rProcessing results from worker %d/%d", i+1, nWorkers)
 			res := <-results

From 3134fcb54ac72505e9af66ce9ab403e430a11a00 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 26 Jun 2020 23:08:06 +0200
Subject: [PATCH 07/47] Streaming mean and variance

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 74 +++++++++++++++++++++++++++++++--------
 1 file changed, 59 insertions(+), 15 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index a6e8de3a4..8f423437b 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -228,13 +228,54 @@ func compStats(vals []float32) (float32, float32) {
 	return av, float32(math.Sqrt(float64(varsum / float32(len(vals)))))
 }
 
-func tallyGasCharges(charges map[string][]float32, et *types.ExecutionTrace) {
+type stats struct {
+	count float64
+	mean  float64
+	dSqr  float64
+}
+
+func (s *stats) AddPoint(value float64) {
+	s.count++
+	meanDiff := (value - s.mean) / s.count
+	newMean := s.mean + meanDiff
+
+	dSqrtInc := (value - newMean) * (value - s.mean)
+	s.dSqr += dSqrtInc
+	s.mean = newMean
+}
+
+func (s *stats) variance() float64 {
+	return s.dSqr / (s.count - 1)
+}
+
+func (s1 *stats) Combine(s2 *stats) {
+	if s1.count == 0 {
+		*s1 = *s2
+	}
+	newCount := s1.count + s2.count
+	newMean := s1.count*s1.mean + s2.count*s2.mean
+	newMean /= newCount
+	newVar := s1.count * (s1.variance() + (s1.mean-newMean)*(s1.mean-newMean))
+	newVar += s2.count * (s2.variance() + (s2.mean-newMean)*(s2.mean-newMean))
+	newVar /= newCount
+	s1.count = newCount
+	s1.mean = newMean
+	s1.dSqr = newVar * (newCount - 1)
+}
+
+func tallyGasCharges(charges map[string]*stats, et *types.ExecutionTrace) {
 	for _, gc := range et.GasCharges {
 
 		compGas := gc.ComputeGas + gc.VirtualComputeGas
-		ratio := float32(compGas) / float32(gc.TimeTaken.Nanoseconds())
+		ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
 
-		charges[gc.Name] = append(charges[gc.Name], 1/(ratio/GasPerNs))
+		s := charges[gc.Name]
+		if s == nil {
+			s = new(stats)
+			charges[gc.Name] = s
+		}
+
+		s.AddPoint(ratio)
 		//fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs))
 		for _, sub := range et.Subcalls {
 			tallyGasCharges(charges, &sub)
@@ -265,7 +306,7 @@ var importAnalyzeCmd = &cli.Command{
 		tseIn := make(chan TipSetExec, 2*nWorkers)
 		type result struct {
 			totalTime       time.Duration
-			chargeDeltas    map[string][]float32
+			chargeStats     map[string]*stats
 			expensiveInvocs []Invocation
 		}
 
@@ -273,7 +314,7 @@ var importAnalyzeCmd = &cli.Command{
 
 		for i := 0; i < nWorkers; i++ {
 			go func() {
-				chargeDeltas := make(map[string][]float32)
+				chargeStats := make(map[string]*stats)
 				var totalTime time.Duration
 				var expensiveInvocs []Invocation
 				var leastExpensiveInvoc = time.Duration(0)
@@ -283,7 +324,7 @@ var importAnalyzeCmd = &cli.Command{
 					if !ok {
 						results <- result{
 							totalTime:       totalTime,
-							chargeDeltas:    chargeDeltas,
+							chargeStats:     chargeStats,
 							expensiveInvocs: expensiveInvocs,
 						}
 						return
@@ -298,7 +339,7 @@ var importAnalyzeCmd = &cli.Command{
 							})
 						}
 
-						tallyGasCharges(chargeDeltas, &inv.ExecutionTrace)
+						tallyGasCharges(chargeStats, &inv.ExecutionTrace)
 					}
 					sort.Slice(expensiveInvocs, func(i, j int) bool {
 						return expensiveInvocs[i].Invoc.Duration > expensiveInvocs[j].Invoc.Duration
@@ -340,29 +381,32 @@ var importAnalyzeCmd = &cli.Command{
 		var invocs []Invocation
 		var totalTime time.Duration
 		var keys []string
-		var chargeDeltas = make(map[string][]float32)
+		var charges = make(map[string]*stats)
 		for i := 0; i < nWorkers; i++ {
 			fmt.Printf("\rProcessing results from worker %d/%d", i+1, nWorkers)
 			res := <-results
 			invocs = append(invocs, res.expensiveInvocs...)
-			for k, v := range res.chargeDeltas {
-				chargeDeltas[k] = append(chargeDeltas[k], v...)
+			for k, v := range res.chargeStats {
+				s := charges[k]
+				if s == nil {
+					s = new(stats)
+					charges[k] = s
+				}
+				s.Combine(v)
 			}
 			totalTime += res.totalTime
 		}
 
 		fmt.Printf("\nCollecting gas keys\n")
-		for k := range chargeDeltas {
+		for k := range charges {
 			keys = append(keys, k)
 		}
 
 		fmt.Println("Gas Price Deltas")
 		sort.Strings(keys)
 		for _, k := range keys {
-			vals := chargeDeltas[k]
-			av, stdev := compStats(vals)
-
-			fmt.Printf("%s: incr by %f (%f)\n", k, av, stdev)
+			s := charges[k]
+			fmt.Printf("%s: incr by %f (%f)\n", k, s.mean, math.Sqrt(s.variance()))
 		}
 
 		sort.Slice(invocs, func(i, j int) bool {

From 9712bbc854b0ff65540c9ab4761c817715f25879 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 26 Jun 2020 23:38:06 +0200
Subject: [PATCH 08/47] Decode in workers

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 44 +++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 8f423437b..bb7b229c6 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -300,10 +301,9 @@ var importAnalyzeCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-		dec := json.NewDecoder(fi)
 
 		const nWorkers = 16
-		tseIn := make(chan TipSetExec, 2*nWorkers)
+		jsonIn := make(chan []byte, 2*nWorkers)
 		type result struct {
 			totalTime       time.Duration
 			chargeStats     map[string]*stats
@@ -316,11 +316,15 @@ var importAnalyzeCmd = &cli.Command{
 			go func() {
 				chargeStats := make(map[string]*stats)
 				var totalTime time.Duration
-				var expensiveInvocs []Invocation
+				const invocsKeep = 32
+				var expensiveInvocs = make([]Invocation, 0, 8*invocsKeep)
 				var leastExpensiveInvoc = time.Duration(0)
 
 				for {
-					tse, ok := <-tseIn
+					b, ok := <-jsonIn
+					var tse TipSetExec
+
+					json.Unmarshal(b, &tse)
 					if !ok {
 						results <- result{
 							totalTime:       totalTime,
@@ -341,10 +345,11 @@ var importAnalyzeCmd = &cli.Command{
 
 						tallyGasCharges(chargeStats, &inv.ExecutionTrace)
 					}
-					sort.Slice(expensiveInvocs, func(i, j int) bool {
-						return expensiveInvocs[i].Invoc.Duration > expensiveInvocs[j].Invoc.Duration
-					})
-					if len(expensiveInvocs) != 0 {
+					if len(expensiveInvocs) > 4*invocsKeep {
+						sort.Slice(expensiveInvocs, func(i, j int) bool {
+							log.Warnf("i: %v, j: %v", expensiveInvocs[i], expensiveInvocs[j])
+							return expensiveInvocs[i].Invoc.Duration > expensiveInvocs[j].Invoc.Duration
+						})
 						leastExpensiveInvoc = expensiveInvocs[len(expensiveInvocs)-1].Invoc.Duration
 						n := 30
 						if len(expensiveInvocs) < n {
@@ -357,24 +362,23 @@ var importAnalyzeCmd = &cli.Command{
 		}
 
 		var totalTipsets int64
+		reader := bufio.NewReader(fi)
 		for {
-			var tse TipSetExec
-			if err := dec.Decode(&tse); err != nil {
-				if err != io.EOF {
-					if e, ok := err.(*json.SyntaxError); ok {
-						log.Warnf("syntax error at byte offset %d", e.Offset)
-					}
-					return err
+			b, err := reader.ReadBytes('\n')
+			if err != nil && err != io.EOF {
+				if e, ok := err.(*json.SyntaxError); ok {
+					log.Warnf("syntax error at byte offset %d", e.Offset)
 				}
-				break
+				return err
 			}
 			totalTipsets++
-			tseIn <- tse
-			if totalTipsets%10 == 0 {
-				fmt.Printf("\rProcessed %d tipsets", totalTipsets)
+			jsonIn <- b
+			fmt.Printf("\rProcessed %d tipsets", totalTipsets)
+			if err == io.EOF {
+				break
 			}
 		}
-		close(tseIn)
+		close(jsonIn)
 		fmt.Printf("\n")
 		fmt.Printf("Collecting results\n")
 

From b5ae3c055b727fc7e0e17d549856dabbb82c67a3 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 26 Jun 2020 23:38:54 +0200
Subject: [PATCH 09/47] Remove warning

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index bb7b229c6..a8d962ff0 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -347,7 +347,6 @@ var importAnalyzeCmd = &cli.Command{
 					}
 					if len(expensiveInvocs) > 4*invocsKeep {
 						sort.Slice(expensiveInvocs, func(i, j int) bool {
-							log.Warnf("i: %v, j: %v", expensiveInvocs[i], expensiveInvocs[j])
 							return expensiveInvocs[i].Invoc.Duration > expensiveInvocs[j].Invoc.Duration
 						})
 						leastExpensiveInvoc = expensiveInvocs[len(expensiveInvocs)-1].Invoc.Duration

From 49ddcdb4b04d662702ae6b7886e9d93310de2c76 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 01:09:35 +0200
Subject: [PATCH 10/47] Don't pass pointers

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index a8d962ff0..4f1f87c4f 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -264,7 +264,7 @@ func (s1 *stats) Combine(s2 *stats) {
 	s1.dSqr = newVar * (newCount - 1)
 }
 
-func tallyGasCharges(charges map[string]*stats, et *types.ExecutionTrace) {
+func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 	for _, gc := range et.GasCharges {
 
 		compGas := gc.ComputeGas + gc.VirtualComputeGas
@@ -279,7 +279,7 @@ func tallyGasCharges(charges map[string]*stats, et *types.ExecutionTrace) {
 		s.AddPoint(ratio)
 		//fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs))
 		for _, sub := range et.Subcalls {
-			tallyGasCharges(charges, &sub)
+			tallyGasCharges(charges, sub)
 		}
 	}
 
@@ -343,7 +343,7 @@ var importAnalyzeCmd = &cli.Command{
 							})
 						}
 
-						tallyGasCharges(chargeStats, &inv.ExecutionTrace)
+						tallyGasCharges(chargeStats, inv.ExecutionTrace)
 					}
 					if len(expensiveInvocs) > 4*invocsKeep {
 						sort.Slice(expensiveInvocs, func(i, j int) bool {

From e5a7cb635c3f3c91a6961ff00bc82abb4d33fc9f Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 01:12:43 +0200
Subject: [PATCH 11/47] Siwtch back to float64

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 4f1f87c4f..76c05135b 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -211,22 +211,22 @@ func countGasCosts(et *types.ExecutionTrace) (int64, int64) {
 	return cgas, vgas
 }
 
-func compStats(vals []float32) (float32, float32) {
-	var sum float32
+func compStats(vals []float64) (float64, float64) {
+	var sum float64
 
 	for _, v := range vals {
 		sum += v
 	}
 
-	av := sum / float32(len(vals))
+	av := sum / float64(len(vals))
 
-	var varsum float32
+	var varsum float64
 	for _, v := range vals {
 		delta := av - v
 		varsum += delta * delta
 	}
 
-	return av, float32(math.Sqrt(float64(varsum / float32(len(vals)))))
+	return av, float64(math.Sqrt(float64(varsum / float64(len(vals)))))
 }
 
 type stats struct {

From 78ee64d9f8bcf69ad6eca2cf7c720ea042fa8df7 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 01:18:06 +0200
Subject: [PATCH 12/47] Fix stupid mistake

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 76c05135b..a9000f0da 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -278,11 +278,10 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 
 		s.AddPoint(ratio)
 		//fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs))
-		for _, sub := range et.Subcalls {
-			tallyGasCharges(charges, sub)
-		}
 	}
-
+	for _, sub := range et.Subcalls {
+		tallyGasCharges(charges, sub)
+	}
 }
 
 var importAnalyzeCmd = &cli.Command{

From 01e230e810eaf6caf6f7ba5e9eb13f8d3f5c7b78 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 01:25:02 +0200
Subject: [PATCH 13/47] Fix ratio

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index a9000f0da..aa3cd68e1 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -276,7 +276,7 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 			charges[gc.Name] = s
 		}
 
-		s.AddPoint(ratio)
+		s.AddPoint(1 / ratio)
 		//fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs))
 	}
 	for _, sub := range et.Subcalls {

From 01ac4e5da3770442202b86e447da009bdf295065 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 16 Jun 2020 18:25:48 +0200
Subject: [PATCH 14/47] Gas on actor exec

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/gas.go | 14 ++++++++++++++
 chain/vm/vm.go  |  5 +++++
 2 files changed, 19 insertions(+)

diff --git a/chain/vm/gas.go b/chain/vm/gas.go
index e6bde25bf..d0e0c46ed 100644
--- a/chain/vm/gas.go
+++ b/chain/vm/gas.go
@@ -143,30 +143,40 @@ func (ps pricedSyscalls) VerifySignature(signature crypto.Signature, signer addr
 		return err
 	}
 	ps.chargeGas(c)
+	defer ps.chargeGas(gasOnActorExec)
+
 	return ps.under.VerifySignature(signature, signer, plaintext)
 }
 
 // Hashes input data using blake2b with 256 bit output.
 func (ps pricedSyscalls) HashBlake2b(data []byte) [32]byte {
 	ps.chargeGas(ps.pl.OnHashing(len(data)))
+	defer ps.chargeGas(gasOnActorExec)
+
 	return ps.under.HashBlake2b(data)
 }
 
 // Computes an unsealed sector CID (CommD) from its constituent piece CIDs (CommPs) and sizes.
 func (ps pricedSyscalls) ComputeUnsealedSectorCID(reg abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) {
 	ps.chargeGas(ps.pl.OnComputeUnsealedSectorCid(reg, pieces))
+	defer ps.chargeGas(gasOnActorExec)
+
 	return ps.under.ComputeUnsealedSectorCID(reg, pieces)
 }
 
 // Verifies a sector seal proof.
 func (ps pricedSyscalls) VerifySeal(vi abi.SealVerifyInfo) error {
 	ps.chargeGas(ps.pl.OnVerifySeal(vi))
+	defer ps.chargeGas(gasOnActorExec)
+
 	return ps.under.VerifySeal(vi)
 }
 
 // Verifies a proof of spacetime.
 func (ps pricedSyscalls) VerifyPoSt(vi abi.WindowPoStVerifyInfo) error {
 	ps.chargeGas(ps.pl.OnVerifyPost(vi))
+	defer ps.chargeGas(gasOnActorExec)
+
 	return ps.under.VerifyPoSt(vi)
 }
 
@@ -182,6 +192,8 @@ func (ps pricedSyscalls) VerifyPoSt(vi abi.WindowPoStVerifyInfo) error {
 // Returns nil and an error if the headers don't prove a fault.
 func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte) (*runtime.ConsensusFault, error) {
 	ps.chargeGas(ps.pl.OnVerifyConsensusFault())
+	defer ps.chargeGas(gasOnActorExec)
+
 	return ps.under.VerifyConsensusFault(h1, h2, extra)
 }
 
@@ -189,6 +201,7 @@ func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]abi.SealVeri
 	var gasChargeSum GasCharge
 	gasChargeSum.Name = "BatchVerifySeals"
 	ps.chargeGas(gasChargeSum) // TODO: this is only called by the cron actor. Should we even charge gas?
+	defer ps.chargeGas(gasOnActorExec)
 
 	for _, svis := range inp {
 		for _, svi := range svis {
@@ -196,5 +209,6 @@ func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]abi.SealVeri
 			ps.chargeGas(newGasCharge("BatchVerifySingle", 0, 0).WithVirtual(ch.VirtualCompute+ch.ComputeGas, 0))
 		}
 	}
+
 	return ps.under.BatchVerifySeals(inp)
 }
diff --git a/chain/vm/vm.go b/chain/vm/vm.go
index 9a4c9991d..e97d963c8 100644
--- a/chain/vm/vm.go
+++ b/chain/vm/vm.go
@@ -35,6 +35,7 @@ import (
 )
 
 var log = logging.Logger("vm")
+var gasOnActorExec = newGasCharge("OnActorExec", 0, 0)
 
 // ResolveToKeyAddr returns the public key type of address (`BLS`/`SECP256K1`) of an account actor identified by `addr`.
 func ResolveToKeyAddr(state types.StateTree, cst cbor.IpldStore, addr address.Address) (address.Address, aerrors.ActorError) {
@@ -68,11 +69,13 @@ type gasChargingBlocks struct {
 }
 
 func (bs *gasChargingBlocks) Get(c cid.Cid) (block.Block, error) {
+	bs.chargeGas(newGasCharge("OnIpldGetStart", 0, 0))
 	blk, err := bs.under.Get(c)
 	if err != nil {
 		return nil, aerrors.Escalate(err, "failed to get block from blockstore")
 	}
 	bs.chargeGas(bs.pricelist.OnIpldGet(len(blk.RawData())))
+	bs.chargeGas(gasOnActorExec)
 
 	return blk, nil
 }
@@ -83,6 +86,7 @@ func (bs *gasChargingBlocks) Put(blk block.Block) error {
 	if err := bs.under.Put(blk); err != nil {
 		return aerrors.Escalate(err, "failed to write data to disk")
 	}
+	bs.chargeGas(gasOnActorExec)
 	return nil
 }
 
@@ -231,6 +235,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
 
 		if msg.Method != 0 {
 			var ret []byte
+			_ = rt.chargeGasSafe(gasOnActorExec)
 			ret, err := vm.Invoke(toActor, rt, msg.Method, msg.Params)
 			return ret, err
 		}

From a2fd0aad41a52447a551d35cdb4d9a12ff60235b Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 16 Jun 2020 19:29:33 +0200
Subject: [PATCH 15/47] More on OnActorExec

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/runtime.go | 4 +++-
 chain/vm/vm.go      | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go
index 595664de1..6de8fec9b 100644
--- a/chain/vm/runtime.go
+++ b/chain/vm/runtime.go
@@ -264,6 +264,7 @@ func (rt *Runtime) CreateActor(codeID cid.Cid, address address.Address) {
 	if err != nil {
 		panic(aerrors.Fatalf("creating actor entry: %v", err))
 	}
+	_ = rt.chargeGasSafe(gasOnActorExec)
 }
 
 func (rt *Runtime) DeleteActor(addr address.Address) {
@@ -284,10 +285,10 @@ func (rt *Runtime) DeleteActor(addr address.Address) {
 	if err := rt.state.DeleteActor(rt.Message().Receiver()); err != nil {
 		panic(aerrors.Fatalf("failed to delete actor: %s", err))
 	}
+	_ = rt.chargeGasSafe(gasOnActorExec)
 }
 
 func (rt *Runtime) Syscalls() vmr.Syscalls {
-	// TODO: Make sure this is wrapped in something that charges gas for each of the calls
 	return rt.sys
 }
 
@@ -367,6 +368,7 @@ func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMars
 		log.Warnf("vmctx send failed: to: %s, method: %d: ret: %d, err: %s", to, method, ret, err)
 		return nil, err.RetCode()
 	}
+	_ = rt.chargeGasSafe(gasOnActorExec)
 	return &dumbWrapperType{ret}, 0
 }
 
diff --git a/chain/vm/vm.go b/chain/vm/vm.go
index e97d963c8..45114c4c5 100644
--- a/chain/vm/vm.go
+++ b/chain/vm/vm.go
@@ -237,6 +237,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
 			var ret []byte
 			_ = rt.chargeGasSafe(gasOnActorExec)
 			ret, err := vm.Invoke(toActor, rt, msg.Method, msg.Params)
+			_ = rt.chargeGasSafe(newGasCharge("OnActorExecDone", 0, 0))
 			return ret, err
 		}
 		return nil, nil

From 9cd34509ee1c2fe975241a9f242b4029b6142ca4 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 01:39:41 +0200
Subject: [PATCH 16/47] Fix error handling of json

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index aa3cd68e1..5cb5e5eb4 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -321,9 +321,6 @@ var importAnalyzeCmd = &cli.Command{
 
 				for {
 					b, ok := <-jsonIn
-					var tse TipSetExec
-
-					json.Unmarshal(b, &tse)
 					if !ok {
 						results <- result{
 							totalTime:       totalTime,
@@ -333,6 +330,13 @@ var importAnalyzeCmd = &cli.Command{
 						return
 					}
 
+					var tse TipSetExec
+					err := json.Unmarshal(b, &tse)
+					if err != nil {
+						log.Warnf("error unmarshaling tipset: %+v", err)
+						continue
+					}
+
 					totalTime += tse.Duration
 					for _, inv := range tse.Trace {
 						if inv.Duration > leastExpensiveInvoc {

From 4f60c1afe737f932552169c23645f44823ee131b Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 01:49:01 +0200
Subject: [PATCH 17/47] If something costs 0 gas, make it 1.

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 5cb5e5eb4..d0250a73c 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -268,6 +268,10 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 	for _, gc := range et.GasCharges {
 
 		compGas := gc.ComputeGas + gc.VirtualComputeGas
+		if compGas == 0 {
+			compGas = 1
+		}
+
 		ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
 
 		s := charges[gc.Name]

From b8ab549bcb6518cb4f3627fedc774a1e66e2455d Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 13:38:11 +0200
Subject: [PATCH 18/47] Cleanup lint

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index d0250a73c..a28878392 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -226,7 +226,7 @@ func compStats(vals []float64) (float64, float64) {
 		varsum += delta * delta
 	}
 
-	return av, float64(math.Sqrt(float64(varsum / float64(len(vals)))))
+	return av, math.Sqrt(varsum / float64(len(vals)))
 }
 
 type stats struct {

From 052fc35a9144d3a1df60a58dfa4594e3b45c3bd4 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 13:55:29 +0200
Subject: [PATCH 19/47] Maybe avoid NaNs

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index a28878392..9de4e1e7b 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -253,6 +253,9 @@ func (s1 *stats) Combine(s2 *stats) {
 	if s1.count == 0 {
 		*s1 = *s2
 	}
+	if s2.count == 0 {
+		return
+	}
 	newCount := s1.count + s2.count
 	newMean := s1.count*s1.mean + s2.count*s2.mean
 	newMean /= newCount
@@ -273,6 +276,10 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 		}
 
 		ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
+		ratio = 1 / ratio
+		if math.IsNaN(ratio) {
+			log.Errorf("NaN: comGas: %f, taken: %d", compGas, gc.TimeTaken.Nanoseconds())
+		}
 
 		s := charges[gc.Name]
 		if s == nil {
@@ -280,7 +287,7 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 			charges[gc.Name] = s
 		}
 
-		s.AddPoint(1 / ratio)
+		s.AddPoint(ratio)
 		//fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs))
 	}
 	for _, sub := range et.Subcalls {

From 369ec8221f0f689268628114133e53a4a03042fc Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 13:59:38 +0200
Subject: [PATCH 20/47] Improve edge case handling

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 9de4e1e7b..c8b61a46f 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -252,10 +252,21 @@ func (s *stats) variance() float64 {
 func (s1 *stats) Combine(s2 *stats) {
 	if s1.count == 0 {
 		*s1 = *s2
+		return
 	}
 	if s2.count == 0 {
 		return
 	}
+	if s1.count == 1 {
+		s2.AddPoint(s1.mean)
+		*s1 = *s2
+		return
+	}
+	if s2.count == 1 {
+		s1.AddPoint(s2.mean)
+		return
+	}
+
 	newCount := s1.count + s2.count
 	newMean := s1.count*s1.mean + s2.count*s2.mean
 	newMean /= newCount

From 1bc9fbca20c87cb38f9d21f59e20357b4b65e98c Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 14:04:20 +0200
Subject: [PATCH 21/47] Ignore virtual

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index c8b61a46f..edc3abeac 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -281,7 +281,7 @@ func (s1 *stats) Combine(s2 *stats) {
 func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 	for _, gc := range et.GasCharges {
 
-		compGas := gc.ComputeGas + gc.VirtualComputeGas
+		compGas := gc.ComputeGas
 		if compGas == 0 {
 			compGas = 1
 		}

From a6417dc24b7770dc528106ba398ce0f40c8b002e Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 14:39:42 +0200
Subject: [PATCH 22/47] Add stats test and print observation count

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go     |  2 +-
 cmd/lotus-bench/stats_test.go | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 cmd/lotus-bench/stats_test.go

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index edc3abeac..f6b85b557 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -434,7 +434,7 @@ var importAnalyzeCmd = &cli.Command{
 		sort.Strings(keys)
 		for _, k := range keys {
 			s := charges[k]
-			fmt.Printf("%s: incr by %f (%f)\n", k, s.mean, math.Sqrt(s.variance()))
+			fmt.Printf("%s: incr by %f (stddev: %f, count: %f)\n", k, s.mean, math.Sqrt(s.variance()), s.count)
 		}
 
 		sort.Slice(invocs, func(i, j int) bool {
diff --git a/cmd/lotus-bench/stats_test.go b/cmd/lotus-bench/stats_test.go
new file mode 100644
index 000000000..851dc71af
--- /dev/null
+++ b/cmd/lotus-bench/stats_test.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"math"
+	"math/rand"
+	"testing"
+)
+
+func TestStats(t *testing.T) {
+	N := 16
+	ss := make([]*stats, N)
+	for i := 0; i < N; i++ {
+		ss[i] = &stats{}
+		maxJ := rand.Intn(1000)
+		for j := 0; j < maxJ; j++ {
+			ss[i].AddPoint(rand.NormFloat64()*5 + 500)
+			ss[i].AddPoint(rand.NormFloat64()*5 + 1000)
+		}
+		t.Logf("mean: %f, stddev: %f, count %f", ss[i].mean, math.Sqrt(ss[i].variance()), ss[i].count)
+	}
+	out := &stats{}
+	for i := 0; i < N; i++ {
+		out.Combine(ss[i])
+		t.Logf("combine: mean: %f, stddev: %f", out.mean, math.Sqrt(out.variance()))
+	}
+}

From 4db61e71da1398bafa0aa8e7256ec6437381988e Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sat, 27 Jun 2020 20:32:14 +0200
Subject: [PATCH 23/47] Reduce caller depth

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/runtime.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go
index 6de8fec9b..c1581dcd9 100644
--- a/chain/vm/runtime.go
+++ b/chain/vm/runtime.go
@@ -525,7 +525,7 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) {
 
 func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError {
 	toUse := gas.Total()
-	var callers [10]uintptr
+	var callers [4]uintptr
 	cout := gruntime.Callers(2+skip, callers[:])
 
 	now := time.Now()

From 909440553779b1b7873298b187095aa6f489c456 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sun, 28 Jun 2020 18:41:03 +0200
Subject: [PATCH 24/47] Try without recurson

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index f6b85b557..6caff50a7 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -360,6 +360,7 @@ var importAnalyzeCmd = &cli.Command{
 					}
 
 					totalTime += tse.Duration
+					var execStack = make([]types.ExecutionTrace, 0, 100)
 					for _, inv := range tse.Trace {
 						if inv.Duration > leastExpensiveInvoc {
 							expensiveInvocs = append(expensiveInvocs, Invocation{
@@ -367,8 +368,33 @@ var importAnalyzeCmd = &cli.Command{
 								Invoc:  inv,
 							})
 						}
+						execStack = append(execStack, inv.ExecutionTrace)
+						for len(execStack) != 0 {
+							et := execStack[len(execStack)-1]
+							execStack = execStack[:len(execStack)-1]
+							for _, gc := range et.GasCharges {
 
-						tallyGasCharges(chargeStats, inv.ExecutionTrace)
+								compGas := gc.ComputeGas
+								if compGas == 0 {
+									compGas = 1
+								}
+
+								ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
+								ratio = 1 / ratio
+								if math.IsNaN(ratio) {
+									log.Errorf("NaN: comGas: %f, taken: %d", compGas, gc.TimeTaken.Nanoseconds())
+								}
+
+								s := chargeStats[gc.Name]
+								if s == nil {
+									s = new(stats)
+									chargeStats[gc.Name] = s
+								}
+
+								s.AddPoint(ratio)
+							}
+							execStack = append(execStack, et.Subcalls...)
+						}
 					}
 					if len(expensiveInvocs) > 4*invocsKeep {
 						sort.Slice(expensiveInvocs, func(i, j int) bool {

From 3a9f1bf9a2a25fc592902eb7276f4335ce61ecb2 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Sun, 28 Jun 2020 18:47:16 +0200
Subject: [PATCH 25/47] Revert "Try without recurson"

This reverts commit 76e3663b2b1f1b58f9097f45124add4fa7718199.

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 28 +---------------------------
 1 file changed, 1 insertion(+), 27 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 6caff50a7..f6b85b557 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -360,7 +360,6 @@ var importAnalyzeCmd = &cli.Command{
 					}
 
 					totalTime += tse.Duration
-					var execStack = make([]types.ExecutionTrace, 0, 100)
 					for _, inv := range tse.Trace {
 						if inv.Duration > leastExpensiveInvoc {
 							expensiveInvocs = append(expensiveInvocs, Invocation{
@@ -368,33 +367,8 @@ var importAnalyzeCmd = &cli.Command{
 								Invoc:  inv,
 							})
 						}
-						execStack = append(execStack, inv.ExecutionTrace)
-						for len(execStack) != 0 {
-							et := execStack[len(execStack)-1]
-							execStack = execStack[:len(execStack)-1]
-							for _, gc := range et.GasCharges {
 
-								compGas := gc.ComputeGas
-								if compGas == 0 {
-									compGas = 1
-								}
-
-								ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
-								ratio = 1 / ratio
-								if math.IsNaN(ratio) {
-									log.Errorf("NaN: comGas: %f, taken: %d", compGas, gc.TimeTaken.Nanoseconds())
-								}
-
-								s := chargeStats[gc.Name]
-								if s == nil {
-									s = new(stats)
-									chargeStats[gc.Name] = s
-								}
-
-								s.AddPoint(ratio)
-							}
-							execStack = append(execStack, et.Subcalls...)
-						}
+						tallyGasCharges(chargeStats, inv.ExecutionTrace)
 					}
 					if len(expensiveInvocs) > 4*invocsKeep {
 						sort.Slice(expensiveInvocs, func(i, j int) bool {

From cd2595da6732ea0c40fcbcb4280f8dd0cc409bd2 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Mon, 29 Jun 2020 18:46:50 +0200
Subject: [PATCH 26/47] Adjust gas

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/gas_v0.go | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go
index 072c1f140..d86edbab8 100644
--- a/chain/vm/gas_v0.go
+++ b/chain/vm/gas_v0.go
@@ -86,12 +86,12 @@ var _ Pricelist = (*pricelistV0)(nil)
 
 // OnChainMessage returns the gas used for storing a message of a given size in the chain.
 func (pl *pricelistV0) OnChainMessage(msgSize int) GasCharge {
-	return newGasCharge("OnChainMessage", 0, pl.onChainMessageBase+pl.onChainMessagePerByte*int64(msgSize))
+	return newGasCharge("OnChainMessage", 0, pl.onChainMessageBase+pl.onChainMessagePerByte*int64(msgSize)).WithVirtual(7402, 0)
 }
 
 // OnChainReturnValue returns the gas used for storing the response of a message in the chain.
 func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge {
-	return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte)
+	return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte).WithVirtual(11160, 0)
 }
 
 // OnMethodInvocation returns the gas used when invoking a method.
@@ -103,22 +103,22 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M
 	if methodNum != builtin.MethodSend {
 		ret += pl.sendInvokeMethod
 	}
-	return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(ret*15000, 0)
+	return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(ret*4600, 0)
 }
 
 // OnIpldGet returns the gas used for storing an object
 func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge {
-	return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).WithExtra(dataSize).WithVirtual(pl.ipldGetBase*13750+(pl.ipldGetPerByte*100), 0)
+	return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).WithExtra(dataSize)
 }
 
 // OnIpldPut returns the gas used for storing an object
 func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge {
-	return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize).WithVirtual(pl.ipldPutBase*8700+(pl.ipldPutPerByte*100), 0)
+	return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize).WithVirtual(475*(pl.ipldPutBase+int64(dataSize)*pl.ipldPutPerByte), 0)
 }
 
 // OnCreateActor returns the gas used for creating an actor
 func (pl *pricelistV0) OnCreateActor() GasCharge {
-	return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra)
+	return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra).WithVirtual(162*pl.createActorBase, 0)
 }
 
 // OnDeleteActor returns the gas used for deleting an actor
@@ -127,6 +127,7 @@ func (pl *pricelistV0) OnDeleteActor() GasCharge {
 }
 
 // OnVerifySignature
+
 func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize int) (GasCharge, error) {
 	costFn, ok := pl.verifySignature[sigType]
 	if !ok {
@@ -138,13 +139,13 @@ func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize in
 
 // OnHashing
 func (pl *pricelistV0) OnHashing(dataSize int) GasCharge {
-	return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0)
+	return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0).WithExtra(dataSize).WithVirtual(256*(pl.hashingBase+int64(dataSize)*pl.hashingPerByte), 0)
 }
 
 // OnComputeUnsealedSectorCid
 func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge {
 	// TODO: this needs more cost tunning, check with @lotus
-	return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0).WithVirtual(pl.computeUnsealedSectorCidBase*24500, 0)
+	return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0).WithVirtual(pl.computeUnsealedSectorCidBase*405, 0)
 }
 
 // OnVerifySeal
@@ -156,10 +157,10 @@ func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) GasCharge {
 // OnVerifyPost
 func (pl *pricelistV0) OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge {
 	// TODO: this needs more cost tunning, check with @lotus
-	return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0)
+	return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0).WithVirtual(pl.verifyPostBase*385100, 0)
 }
 
 // OnVerifyConsensusFault
 func (pl *pricelistV0) OnVerifyConsensusFault() GasCharge {
-	return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0)
+	return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0).WithVirtual(pl.verifyConsensusFault*5466, 0)
 }

From ca7e5f5e16d6acccd21a501a322a9e49d3c5d8dc Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Mon, 29 Jun 2020 18:56:27 +0200
Subject: [PATCH 27/47] Set gas to 1

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index f6b85b557..3d994f3b6 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -281,16 +281,24 @@ func (s1 *stats) Combine(s2 *stats) {
 func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 	for _, gc := range et.GasCharges {
 
-		compGas := gc.ComputeGas
+		compGas := gc.VirtualComputeGas
 		if compGas == 0 {
 			compGas = 1
 		}
+		compGas = 1
 
 		ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
 		ratio = 1 / ratio
 		if math.IsNaN(ratio) {
 			log.Errorf("NaN: comGas: %f, taken: %d", compGas, gc.TimeTaken.Nanoseconds())
 		}
+		name := gc.Name
+		if eString, ok := gc.Extra.(string); ok {
+			name += "-" + eString
+		} else if eInt, ok := gc.Extra.(float64); ok {
+			// handle scaling
+			_ = eInt
+		}
 
 		s := charges[gc.Name]
 		if s == nil {

From 1e17ea3fffbd53c2fff4eb2f8357eb11095daf0e Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Mon, 29 Jun 2020 19:19:02 +0200
Subject: [PATCH 28/47] Adjust gas values

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/gas_v0.go        | 19 +++++++++++--------
 chain/vm/vm.go            |  2 +-
 cmd/lotus-bench/import.go | 28 ++++++++++++++--------------
 3 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go
index d86edbab8..a0543cb72 100644
--- a/chain/vm/gas_v0.go
+++ b/chain/vm/gas_v0.go
@@ -97,13 +97,16 @@ func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge {
 // OnMethodInvocation returns the gas used when invoking a method.
 func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) GasCharge {
 	ret := pl.sendBase
+	extra := ""
 	if value != abi.NewTokenAmount(0) {
 		ret += pl.sendTransferFunds
+		extra += "t"
 	}
 	if methodNum != builtin.MethodSend {
 		ret += pl.sendInvokeMethod
+		extra += "i"
 	}
-	return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(ret*4600, 0)
+	return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(86315, 0).WithExtra(extra)
 }
 
 // OnIpldGet returns the gas used for storing an object
@@ -113,12 +116,12 @@ func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge {
 
 // OnIpldPut returns the gas used for storing an object
 func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge {
-	return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize).WithVirtual(475*(pl.ipldPutBase+int64(dataSize)*pl.ipldPutPerByte), 0)
+	return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize).WithVirtual(9151, 0)
 }
 
 // OnCreateActor returns the gas used for creating an actor
 func (pl *pricelistV0) OnCreateActor() GasCharge {
-	return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra).WithVirtual(162*pl.createActorBase, 0)
+	return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra).WithVirtual(6549, 0)
 }
 
 // OnDeleteActor returns the gas used for deleting an actor
@@ -139,28 +142,28 @@ func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize in
 
 // OnHashing
 func (pl *pricelistV0) OnHashing(dataSize int) GasCharge {
-	return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0).WithExtra(dataSize).WithVirtual(256*(pl.hashingBase+int64(dataSize)*pl.hashingPerByte), 0)
+	return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0).WithExtra(dataSize).WithVirtual(7881, 0)
 }
 
 // OnComputeUnsealedSectorCid
 func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge {
 	// TODO: this needs more cost tunning, check with @lotus
-	return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0).WithVirtual(pl.computeUnsealedSectorCidBase*405, 0)
+	return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0).WithVirtual(40536, 0)
 }
 
 // OnVerifySeal
 func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) GasCharge {
 	// TODO: this needs more cost tunning, check with @lotus
-	return newGasCharge("OnVerifySeal", pl.verifySealBase, 0).WithVirtual(pl.verifySealBase*177500, 0)
+	return newGasCharge("OnVerifySeal", pl.verifySealBase, 0).WithVirtual(199954003, 0)
 }
 
 // OnVerifyPost
 func (pl *pricelistV0) OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge {
 	// TODO: this needs more cost tunning, check with @lotus
-	return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0).WithVirtual(pl.verifyPostBase*385100, 0)
+	return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0).WithVirtual(269570688, 0)
 }
 
 // OnVerifyConsensusFault
 func (pl *pricelistV0) OnVerifyConsensusFault() GasCharge {
-	return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0).WithVirtual(pl.verifyConsensusFault*5466, 0)
+	return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0).WithVirtual(54665, 0)
 }
diff --git a/chain/vm/vm.go b/chain/vm/vm.go
index 45114c4c5..c76f01dac 100644
--- a/chain/vm/vm.go
+++ b/chain/vm/vm.go
@@ -69,7 +69,7 @@ type gasChargingBlocks struct {
 }
 
 func (bs *gasChargingBlocks) Get(c cid.Cid) (block.Block, error) {
-	bs.chargeGas(newGasCharge("OnIpldGetStart", 0, 0))
+	bs.chargeGas(newGasCharge("OnIpldGetStart", 0, 0).WithVirtual(40175))
 	blk, err := bs.under.Get(c)
 	if err != nil {
 		return nil, aerrors.Escalate(err, "failed to get block from blockstore")
diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 3d994f3b6..f76ef4644 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -280,18 +280,6 @@ func (s1 *stats) Combine(s2 *stats) {
 
 func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 	for _, gc := range et.GasCharges {
-
-		compGas := gc.VirtualComputeGas
-		if compGas == 0 {
-			compGas = 1
-		}
-		compGas = 1
-
-		ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
-		ratio = 1 / ratio
-		if math.IsNaN(ratio) {
-			log.Errorf("NaN: comGas: %f, taken: %d", compGas, gc.TimeTaken.Nanoseconds())
-		}
 		name := gc.Name
 		if eString, ok := gc.Extra.(string); ok {
 			name += "-" + eString
@@ -300,10 +288,22 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 			_ = eInt
 		}
 
-		s := charges[gc.Name]
+		compGas := gc.VirtualComputeGas
+		if compGas == 0 {
+			name += "-zerogas"
+			compGas = 1
+		}
+
+		ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
+		ratio = 1 / ratio
+		if math.IsNaN(ratio) {
+			log.Errorf("NaN: comGas: %f, taken: %d", compGas, gc.TimeTaken.Nanoseconds())
+		}
+
+		s := charges[name]
 		if s == nil {
 			s = new(stats)
-			charges[gc.Name] = s
+			charges[name] = s
 		}
 
 		s.AddPoint(ratio)

From bc88c632d15185bfb88c068775ceeac318faba81 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Mon, 29 Jun 2020 19:23:50 +0200
Subject: [PATCH 29/47] Fix previous commit

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/vm.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/chain/vm/vm.go b/chain/vm/vm.go
index c76f01dac..5c2f7f9ea 100644
--- a/chain/vm/vm.go
+++ b/chain/vm/vm.go
@@ -69,7 +69,7 @@ type gasChargingBlocks struct {
 }
 
 func (bs *gasChargingBlocks) Get(c cid.Cid) (block.Block, error) {
-	bs.chargeGas(newGasCharge("OnIpldGetStart", 0, 0).WithVirtual(40175))
+	bs.chargeGas(newGasCharge("OnIpldGetStart", 0, 0).WithVirtual(40175, 0))
 	blk, err := bs.under.Get(c)
 	if err != nil {
 		return nil, aerrors.Escalate(err, "failed to get block from blockstore")

From 21148033e20fe66ed1df687767bd381dee4222a4 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 00:31:05 +0200
Subject: [PATCH 30/47] Change alg for computing variance

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go     | 29 +++++++++++++----------------
 cmd/lotus-bench/stats_test.go |  6 +++---
 2 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index f76ef4644..ef572b13b 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -232,21 +232,20 @@ func compStats(vals []float64) (float64, float64) {
 type stats struct {
 	count float64
 	mean  float64
-	dSqr  float64
+	m2    float64
 }
 
 func (s *stats) AddPoint(value float64) {
-	s.count++
-	meanDiff := (value - s.mean) / s.count
-	newMean := s.mean + meanDiff
-
-	dSqrtInc := (value - newMean) * (value - s.mean)
-	s.dSqr += dSqrtInc
-	s.mean = newMean
+	// based on https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
+	s.count += 1
+	delta := value - s.mean
+	s.mean += delta / s.count
+	delta2 := value - s.mean
+	s.m2 += delta * delta2
 }
 
 func (s *stats) variance() float64 {
-	return s.dSqr / (s.count - 1)
+	return s.m2 / (s.count - 1)
 }
 
 func (s1 *stats) Combine(s2 *stats) {
@@ -268,14 +267,12 @@ func (s1 *stats) Combine(s2 *stats) {
 	}
 
 	newCount := s1.count + s2.count
-	newMean := s1.count*s1.mean + s2.count*s2.mean
-	newMean /= newCount
-	newVar := s1.count * (s1.variance() + (s1.mean-newMean)*(s1.mean-newMean))
-	newVar += s2.count * (s2.variance() + (s2.mean-newMean)*(s2.mean-newMean))
-	newVar /= newCount
+	delta := s2.mean - s1.mean
+	meanDelta := delta * s2.count / newCount
+	m2 := s1.m2 + s2.m2 + delta*meanDelta*s1.count
 	s1.count = newCount
-	s1.mean = newMean
-	s1.dSqr = newVar * (newCount - 1)
+	s1.mean += meanDelta
+	s1.m2 = m2
 }
 
 func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
diff --git a/cmd/lotus-bench/stats_test.go b/cmd/lotus-bench/stats_test.go
index 851dc71af..6ade8a4e5 100644
--- a/cmd/lotus-bench/stats_test.go
+++ b/cmd/lotus-bench/stats_test.go
@@ -9,12 +9,12 @@ import (
 func TestStats(t *testing.T) {
 	N := 16
 	ss := make([]*stats, N)
+	rng := rand.New(rand.NewSource(1))
 	for i := 0; i < N; i++ {
 		ss[i] = &stats{}
-		maxJ := rand.Intn(1000)
+		maxJ := rng.Intn(1000)
 		for j := 0; j < maxJ; j++ {
-			ss[i].AddPoint(rand.NormFloat64()*5 + 500)
-			ss[i].AddPoint(rand.NormFloat64()*5 + 1000)
+			ss[i].AddPoint(rng.NormFloat64()*5 + 500)
 		}
 		t.Logf("mean: %f, stddev: %f, count %f", ss[i].mean, math.Sqrt(ss[i].variance()), ss[i].count)
 	}

From 2caa7164a6d2d69d29e049db93d58360138faced Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 01:31:09 +0200
Subject: [PATCH 31/47] Midway adding covar

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 141 ++++++++++++++++++++++++++++----------
 1 file changed, 103 insertions(+), 38 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index ef572b13b..0a9f2c72b 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -230,49 +230,120 @@ func compStats(vals []float64) (float64, float64) {
 }
 
 type stats struct {
-	count float64
-	mean  float64
+	timeTaken meanVar
+	gasRatio  meanVar
+
+	extra      *meanVar
+	extraCovar *covar
+}
+
+type covar struct {
+	meanX float64
+	meanY float64
+	c     float64
+	n     float64
 	m2    float64
 }
 
-func (s *stats) AddPoint(value float64) {
+func (cov1 *covar) Covariance() float64 {
+	return cov1.c / (cov1.n - 1)
+}
+
+func (cov1 *covar) AddPoint(x, y float64) {
+	cov1.n += 1
+	dx := x - cov1.meanX
+	cov1.meanX += dx / cov1.n
+
+	dx2 := x - cov1.meanX // compute x variance using partial result for covariance
+	cov1.m2 += dx * dx2
+
+	cov1.meanY += (y - cov1.meanY) / cov1.n
+	cov1.c += dx * (y - cov1.meanY)
+}
+
+func (cov1 *covar) Combine(cov2 *covar) {
+	if cov1.n == 0 {
+		*cov1 = *cov2
+		return
+	}
+	if cov2.n == 0 {
+		return
+	}
+
+	if cov1.n == 1 {
+		cpy := *cov2
+		cpy.AddPoint(cov2.meanX, cov2.meanY)
+		*cov1 = cpy
+		return
+	}
+	if cov2.n == 1 {
+		cov1.AddPoint(cov2.meanX, cov2.meanY)
+	}
+	out := covar{}
+	out.n = cov1.n + cov2.n
+
+	dx := cov1.meanX - cov2.meanX
+	out.meanX = cov1.meanX + dx*cov2.n/out.n
+	out.m2 = cov1.m2 + cov2.m2 + dx*dx*cov1.n*cov2.n/out.n
+
+	dy := cov1.meanY - cov2.meanY
+	out.meanY = cov1.meanY + dy*cov2.n/out.n
+
+	out.c = cov1.c + cov2.c + dx*dy*cov1.n*cov2.n/out.n
+	*cov1 = out
+}
+
+type meanVar struct {
+	n    float64
+	mean float64
+	m2   float64
+}
+
+func (v1 *meanVar) AddPoint(value float64) {
 	// based on https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
-	s.count += 1
-	delta := value - s.mean
-	s.mean += delta / s.count
-	delta2 := value - s.mean
-	s.m2 += delta * delta2
+	v1.n += 1
+	delta := value - v1.mean
+	v1.mean += delta / v1.n
+	delta2 := value - v1.mean
+	v1.m2 += delta * delta2
 }
 
-func (s *stats) variance() float64 {
-	return s.m2 / (s.count - 1)
+func (v1 *meanVar) Variance() float64 {
+	return v1.m2 / (v1.n - 1)
+}
+func (v1 *meanVar) Mean() float64 {
+	return v1.mean
+}
+func (v1 *meanVar) Stddev() float64 {
+	return math.Sqrt(v1.Variance())
 }
 
-func (s1 *stats) Combine(s2 *stats) {
-	if s1.count == 0 {
-		*s1 = *s2
+func (v1 *meanVar) Combine(v2 *meanVar) {
+	if v1.n == 0 {
+		*v1 = *v2
 		return
 	}
-	if s2.count == 0 {
+	if v2.n == 0 {
 		return
 	}
-	if s1.count == 1 {
-		s2.AddPoint(s1.mean)
-		*s1 = *s2
+	if v1.n == 1 {
+		cpy := *v2
+		cpy.AddPoint(v1.mean)
+		*v1 = cpy
 		return
 	}
-	if s2.count == 1 {
-		s1.AddPoint(s2.mean)
+	if v2.n == 1 {
+		v1.AddPoint(v2.mean)
 		return
 	}
 
-	newCount := s1.count + s2.count
-	delta := s2.mean - s1.mean
-	meanDelta := delta * s2.count / newCount
-	m2 := s1.m2 + s2.m2 + delta*meanDelta*s1.count
-	s1.count = newCount
-	s1.mean += meanDelta
-	s1.m2 = m2
+	newCount := v1.n + v2.n
+	delta := v2.mean - v1.mean
+	meanDelta := delta * v2.n / newCount
+	m2 := v1.m2 + v2.m2 + delta*meanDelta*v1.n
+	v1.n = newCount
+	v1.mean += meanDelta
+	v1.m2 = m2
 }
 
 func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
@@ -287,24 +358,17 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 
 		compGas := gc.VirtualComputeGas
 		if compGas == 0 {
-			name += "-zerogas"
 			compGas = 1
 		}
-
-		ratio := float64(compGas) / float64(gc.TimeTaken.Nanoseconds())
-		ratio = 1 / ratio
-		if math.IsNaN(ratio) {
-			log.Errorf("NaN: comGas: %f, taken: %d", compGas, gc.TimeTaken.Nanoseconds())
-		}
-
 		s := charges[name]
 		if s == nil {
 			s = new(stats)
 			charges[name] = s
 		}
+		s.timeTaken.AddPoint(float64(gc.TimeTaken.Nanoseconds()))
 
-		s.AddPoint(ratio)
-		//fmt.Printf("%s: %d, %s: %0.2f\n", gc.Name, compGas, gc.TimeTaken, 1/(ratio/GasPerNs))
+		ratio := float64(gc.TimeTaken.Nanoseconds()) / float64(compGas) * GasPerNs
+		s.gasRatio.AddPoint(ratio)
 	}
 	for _, sub := range et.Subcalls {
 		tallyGasCharges(charges, sub)
@@ -425,7 +489,8 @@ var importAnalyzeCmd = &cli.Command{
 					s = new(stats)
 					charges[k] = s
 				}
-				s.Combine(v)
+				s.timeTaken.Combine(&v.timeTaken)
+				s.gasRatio.Combine(&v.gasRatio)
 			}
 			totalTime += res.totalTime
 		}
@@ -439,7 +504,7 @@ var importAnalyzeCmd = &cli.Command{
 		sort.Strings(keys)
 		for _, k := range keys {
 			s := charges[k]
-			fmt.Printf("%s: incr by %f (stddev: %f, count: %f)\n", k, s.mean, math.Sqrt(s.variance()), s.count)
+			fmt.Printf("%s: incr by %f~%f\n", k, s.gasRatio.mean, s.gasRatio.Stddev())
 		}
 
 		sort.Slice(invocs, func(i, j int) bool {

From 29f5f451b31818897158905b143f7992b77f61c3 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 02:11:54 +0200
Subject: [PATCH 32/47] Add covariance

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go     | 66 ++++++++++++++++++++++++++++++-----
 cmd/lotus-bench/stats_test.go | 33 ++++++++++++++----
 2 files changed, 83 insertions(+), 16 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 0a9f2c72b..f9526e90f 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -249,6 +249,14 @@ func (cov1 *covar) Covariance() float64 {
 	return cov1.c / (cov1.n - 1)
 }
 
+func (cov1 *covar) Variance() float64 {
+	return cov1.m2 / (cov1.n - 1)
+}
+
+func (v1 *covar) Stddev() float64 {
+	return math.Sqrt(v1.Variance())
+}
+
 func (cov1 *covar) AddPoint(x, y float64) {
 	cov1.n += 1
 	dx := x - cov1.meanX
@@ -279,11 +287,12 @@ func (cov1 *covar) Combine(cov2 *covar) {
 	if cov2.n == 1 {
 		cov1.AddPoint(cov2.meanX, cov2.meanY)
 	}
+
 	out := covar{}
 	out.n = cov1.n + cov2.n
 
 	dx := cov1.meanX - cov2.meanX
-	out.meanX = cov1.meanX + dx*cov2.n/out.n
+	out.meanX = cov1.meanX - dx*cov2.n/out.n
 	out.m2 = cov1.m2 + cov2.m2 + dx*dx*cov1.n*cov2.n/out.n
 
 	dy := cov1.meanY - cov2.meanY
@@ -293,6 +302,13 @@ func (cov1 *covar) Combine(cov2 *covar) {
 	*cov1 = out
 }
 
+func (cov1 *covar) A() float64 {
+	return cov1.Covariance() / cov1.Variance()
+}
+func (cov1 *covar) B() float64 {
+	return cov1.meanY - cov1.meanX*cov1.A()
+}
+
 type meanVar struct {
 	n    float64
 	mean float64
@@ -347,15 +363,22 @@ func (v1 *meanVar) Combine(v2 *meanVar) {
 }
 
 func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
-	for _, gc := range et.GasCharges {
+	for i, gc := range et.GasCharges {
 		name := gc.Name
+		if name == "OnIpldGetStart" {
+			continue
+		}
+		tt := float64(gc.TimeTaken.Nanoseconds())
+		if name == "OnIpldGet" {
+			prev := et.GasCharges[i-1]
+			if prev.Name != "OnIpldGetStart" {
+				log.Warn("OnIpldGet without OnIpldGetStart")
+			}
+			tt += float64(prev.TimeTaken.Nanoseconds())
+		}
 		if eString, ok := gc.Extra.(string); ok {
 			name += "-" + eString
-		} else if eInt, ok := gc.Extra.(float64); ok {
-			// handle scaling
-			_ = eInt
 		}
-
 		compGas := gc.VirtualComputeGas
 		if compGas == 0 {
 			compGas = 1
@@ -365,9 +388,19 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 			s = new(stats)
 			charges[name] = s
 		}
-		s.timeTaken.AddPoint(float64(gc.TimeTaken.Nanoseconds()))
 
-		ratio := float64(gc.TimeTaken.Nanoseconds()) / float64(compGas) * GasPerNs
+		if extra, ok := gc.Extra.(float64); ok {
+			if s.extraCovar == nil {
+				s.extraCovar = &covar{}
+				s.extra = &meanVar{}
+			}
+			s.extraCovar.AddPoint(extra, tt)
+			s.extra.AddPoint(extra)
+		}
+
+		s.timeTaken.AddPoint(tt)
+
+		ratio := tt / float64(compGas) * GasPerNs
 		s.gasRatio.AddPoint(ratio)
 	}
 	for _, sub := range et.Subcalls {
@@ -491,6 +524,15 @@ var importAnalyzeCmd = &cli.Command{
 				}
 				s.timeTaken.Combine(&v.timeTaken)
 				s.gasRatio.Combine(&v.gasRatio)
+
+				if v.extra != nil {
+					if s.extra == nil {
+						s.extra = &meanVar{}
+						s.extraCovar = &covar{}
+					}
+					s.extra.Combine(v.extra)
+					s.extraCovar.Combine(v.extraCovar)
+				}
 			}
 			totalTime += res.totalTime
 		}
@@ -504,7 +546,13 @@ var importAnalyzeCmd = &cli.Command{
 		sort.Strings(keys)
 		for _, k := range keys {
 			s := charges[k]
-			fmt.Printf("%s: incr by %f~%f\n", k, s.gasRatio.mean, s.gasRatio.Stddev())
+			fmt.Printf("%s: incr by %f~%f; tt %f~%f\n", k, s.gasRatio.Mean(), s.gasRatio.Stddev(),
+				s.timeTaken.Mean(), s.timeTaken.Stddev())
+			if s.extra != nil {
+				fmt.Printf("\t covar: %f, tt = %f * extra + %f\n", s.extraCovar.Covariance(),
+					s.extraCovar.A(), s.extraCovar.B())
+				fmt.Printf("\t extra: %f~%f\n", s.extra.Mean(), s.extra.Stddev())
+			}
 		}
 
 		sort.Slice(invocs, func(i, j int) bool {
diff --git a/cmd/lotus-bench/stats_test.go b/cmd/lotus-bench/stats_test.go
index 6ade8a4e5..a8cb985a1 100644
--- a/cmd/lotus-bench/stats_test.go
+++ b/cmd/lotus-bench/stats_test.go
@@ -1,26 +1,45 @@
 package main
 
 import (
-	"math"
 	"math/rand"
 	"testing"
 )
 
-func TestStats(t *testing.T) {
+func TestMeanVar(t *testing.T) {
 	N := 16
-	ss := make([]*stats, N)
+	ss := make([]*meanVar, N)
 	rng := rand.New(rand.NewSource(1))
 	for i := 0; i < N; i++ {
-		ss[i] = &stats{}
+		ss[i] = &meanVar{}
 		maxJ := rng.Intn(1000)
 		for j := 0; j < maxJ; j++ {
 			ss[i].AddPoint(rng.NormFloat64()*5 + 500)
 		}
-		t.Logf("mean: %f, stddev: %f, count %f", ss[i].mean, math.Sqrt(ss[i].variance()), ss[i].count)
+		t.Logf("mean: %f, stddev: %f, count %f", ss[i].mean, ss[i].Stddev(), ss[i].n)
 	}
-	out := &stats{}
+	out := &meanVar{}
 	for i := 0; i < N; i++ {
 		out.Combine(ss[i])
-		t.Logf("combine: mean: %f, stddev: %f", out.mean, math.Sqrt(out.variance()))
+		t.Logf("combine: mean: %f, stddev: %f", out.mean, out.Stddev())
+	}
+}
+
+func TestCovar(t *testing.T) {
+	N := 16
+	ss := make([]*covar, N)
+	rng := rand.New(rand.NewSource(1))
+	for i := 0; i < N; i++ {
+		ss[i] = &covar{}
+		maxJ := rng.Intn(1000) + 500
+		for j := 0; j < maxJ; j++ {
+			x := rng.NormFloat64()*5 + 500
+			ss[i].AddPoint(x, x*(rng.NormFloat64()/10+2)-rng.NormFloat64()-1000)
+		}
+		t.Logf("covar: %f, y = %f*x+%f @%.0f", ss[i].Covariance(), ss[i].A(), ss[i].B(), ss[i].n)
+	}
+	out := &covar{}
+	for i := 0; i < N; i++ {
+		out.Combine(ss[i])
+		t.Logf("combine: covar: %f, y = %f*x+%f", ss[i].Covariance(), ss[i].A(), ss[i].B())
 	}
 }

From c68a8f802ee16225179091de465263633a0d2874 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 02:31:39 +0200
Subject: [PATCH 33/47] Adjust gas

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/gas.go    | 11 ++++-------
 chain/vm/gas_v0.go | 32 ++++++++++++++++++++++----------
 chain/vm/vm.go     |  2 +-
 3 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/chain/vm/gas.go b/chain/vm/gas.go
index d0e0c46ed..e1f7952b9 100644
--- a/chain/vm/gas.go
+++ b/chain/vm/gas.go
@@ -200,15 +200,12 @@ func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte
 func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) {
 	var gasChargeSum GasCharge
 	gasChargeSum.Name = "BatchVerifySeals"
-	ps.chargeGas(gasChargeSum) // TODO: this is only called by the cron actor. Should we even charge gas?
-	defer ps.chargeGas(gasOnActorExec)
-
+	count := 0
 	for _, svis := range inp {
-		for _, svi := range svis {
-			ch := ps.pl.OnVerifySeal(svi)
-			ps.chargeGas(newGasCharge("BatchVerifySingle", 0, 0).WithVirtual(ch.VirtualCompute+ch.ComputeGas, 0))
-		}
+		count += len(svis)
 	}
+	ps.chargeGas(gasChargeSum.WithExtra(count)) // TODO: this is only called by the cron actor. Should we even charge gas?
+	defer ps.chargeGas(gasOnActorExec)
 
 	return ps.under.BatchVerifySeals(inp)
 }
diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go
index a0543cb72..0215b7454 100644
--- a/chain/vm/gas_v0.go
+++ b/chain/vm/gas_v0.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/crypto"
 )
@@ -86,20 +87,25 @@ var _ Pricelist = (*pricelistV0)(nil)
 
 // OnChainMessage returns the gas used for storing a message of a given size in the chain.
 func (pl *pricelistV0) OnChainMessage(msgSize int) GasCharge {
-	return newGasCharge("OnChainMessage", 0, pl.onChainMessageBase+pl.onChainMessagePerByte*int64(msgSize)).WithVirtual(7402, 0)
+	return newGasCharge("OnChainMessage", 0, pl.onChainMessageBase+pl.onChainMessagePerByte*int64(msgSize)).WithVirtual(77302, 0)
 }
 
 // OnChainReturnValue returns the gas used for storing the response of a message in the chain.
 func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge {
-	return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte).WithVirtual(11160, 0)
+	return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte).WithVirtual(107294, 0)
 }
 
 // OnMethodInvocation returns the gas used when invoking a method.
 func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) GasCharge {
 	ret := pl.sendBase
 	extra := ""
+
 	if value != abi.NewTokenAmount(0) {
+		// TODO: fix this, it is comparing pointers instead of values
+		// see vv
 		ret += pl.sendTransferFunds
+	}
+	if big.Cmp(value, abi.NewTokenAmount(0)) == 0 {
 		extra += "t"
 	}
 	if methodNum != builtin.MethodSend {
@@ -111,17 +117,19 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M
 
 // OnIpldGet returns the gas used for storing an object
 func (pl *pricelistV0) OnIpldGet(dataSize int) GasCharge {
-	return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).WithExtra(dataSize)
+	return newGasCharge("OnIpldGet", pl.ipldGetBase+int64(dataSize)*pl.ipldGetPerByte, 0).
+		WithExtra(dataSize).WithVirtual(433685, 0)
 }
 
 // OnIpldPut returns the gas used for storing an object
 func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge {
-	return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).WithExtra(dataSize).WithVirtual(9151, 0)
+	return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).
+		WithExtra(dataSize).WithVirtual(88970, 0)
 }
 
 // OnCreateActor returns the gas used for creating an actor
 func (pl *pricelistV0) OnCreateActor() GasCharge {
-	return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra).WithVirtual(6549, 0)
+	return newGasCharge("OnCreateActor", pl.createActorBase, pl.createActorExtra).WithVirtual(65636, 0)
 }
 
 // OnDeleteActor returns the gas used for deleting an actor
@@ -137,18 +145,22 @@ func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize in
 		return GasCharge{}, fmt.Errorf("cost function for signature type %d not supported", sigType)
 	}
 	sigName, _ := sigType.Name()
-	return newGasCharge("OnVerifySignature", costFn(int64(planTextSize)), 0).WithExtra(sigName), nil
+	return newGasCharge("OnVerifySignature", costFn(int64(planTextSize)), 0).
+		WithExtra(map[string]interface{}{
+			"type": sigName,
+			"size": planTextSize,
+		}), nil
 }
 
 // OnHashing
 func (pl *pricelistV0) OnHashing(dataSize int) GasCharge {
-	return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0).WithExtra(dataSize).WithVirtual(7881, 0)
+	return newGasCharge("OnHashing", pl.hashingBase+int64(dataSize)*pl.hashingPerByte, 0).WithExtra(dataSize).WithVirtual(77300, 0)
 }
 
 // OnComputeUnsealedSectorCid
 func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge {
 	// TODO: this needs more cost tunning, check with @lotus
-	return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0).WithVirtual(40536, 0)
+	return newGasCharge("OnComputeUnsealedSectorCid", pl.computeUnsealedSectorCidBase, 0).WithVirtual(382370, 0)
 }
 
 // OnVerifySeal
@@ -160,10 +172,10 @@ func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) GasCharge {
 // OnVerifyPost
 func (pl *pricelistV0) OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge {
 	// TODO: this needs more cost tunning, check with @lotus
-	return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0).WithVirtual(269570688, 0)
+	return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0).WithVirtual(2629471704, 0)
 }
 
 // OnVerifyConsensusFault
 func (pl *pricelistV0) OnVerifyConsensusFault() GasCharge {
-	return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0).WithVirtual(54665, 0)
+	return newGasCharge("OnVerifyConsensusFault", pl.verifyConsensusFault, 0).WithVirtual(551935, 0)
 }
diff --git a/chain/vm/vm.go b/chain/vm/vm.go
index 5c2f7f9ea..45114c4c5 100644
--- a/chain/vm/vm.go
+++ b/chain/vm/vm.go
@@ -69,7 +69,7 @@ type gasChargingBlocks struct {
 }
 
 func (bs *gasChargingBlocks) Get(c cid.Cid) (block.Block, error) {
-	bs.chargeGas(newGasCharge("OnIpldGetStart", 0, 0).WithVirtual(40175, 0))
+	bs.chargeGas(newGasCharge("OnIpldGetStart", 0, 0))
 	blk, err := bs.under.Get(c)
 	if err != nil {
 		return nil, aerrors.Escalate(err, "failed to get block from blockstore")

From 5e7f5611338e3f19fc931e233b05601de3f94d31 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 02:56:58 +0200
Subject: [PATCH 34/47] Support complex extra, fix small bug in covariance

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 37 +++++++++++++++++++++++++++++++------
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index f9526e90f..7d759642e 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -296,7 +296,7 @@ func (cov1 *covar) Combine(cov2 *covar) {
 	out.m2 = cov1.m2 + cov2.m2 + dx*dx*cov1.n*cov2.n/out.n
 
 	dy := cov1.meanY - cov2.meanY
-	out.meanY = cov1.meanY + dy*cov2.n/out.n
+	out.meanY = cov1.meanY - dy*cov2.n/out.n
 
 	out.c = cov1.c + cov2.c + dx*dy*cov1.n*cov2.n/out.n
 	*cov1 = out
@@ -362,6 +362,30 @@ func (v1 *meanVar) Combine(v2 *meanVar) {
 	v1.m2 = m2
 }
 
+func getExtras(ex interface{}) (*string, *float64) {
+	if t, ok := ex.(string); ok {
+		return &t, nil
+	}
+	if size, ok := ex.(float64); ok {
+		return nil, &size
+	}
+	if exMap, ok := ex.(map[string]interface{}); ok {
+		t, tok := exMap["type"].(string)
+		size, sok := exMap["size"].(float64)
+		if tok && sok {
+			return &t, &size
+		}
+		if tok {
+			return &t, nil
+		}
+		if sok {
+			return nil, &size
+		}
+		return nil, nil
+	}
+	return nil, nil
+}
+
 func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 	for i, gc := range et.GasCharges {
 		name := gc.Name
@@ -376,8 +400,9 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 			}
 			tt += float64(prev.TimeTaken.Nanoseconds())
 		}
-		if eString, ok := gc.Extra.(string); ok {
-			name += "-" + eString
+		eType, eSize := getExtras(gc.Extra)
+		if eType != nil {
+			name += "-" + *eType
 		}
 		compGas := gc.VirtualComputeGas
 		if compGas == 0 {
@@ -389,13 +414,13 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 			charges[name] = s
 		}
 
-		if extra, ok := gc.Extra.(float64); ok {
+		if eSize != nil {
 			if s.extraCovar == nil {
 				s.extraCovar = &covar{}
 				s.extra = &meanVar{}
 			}
-			s.extraCovar.AddPoint(extra, tt)
-			s.extra.AddPoint(extra)
+			s.extraCovar.AddPoint(*eSize, tt)
+			s.extra.AddPoint(*eSize)
 		}
 
 		s.timeTaken.AddPoint(tt)

From 9ccd25b0725572e8c0dbfef165027197d261a102 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 14:09:03 +0200
Subject: [PATCH 35/47] Update gas values, compute correlation coefficient

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/gas.go           |  7 ++---
 chain/vm/gas_v0.go        | 20 +++++++++++++--
 cmd/lotus-bench/import.go | 54 ++++++++++++++++++++++++---------------
 3 files changed, 55 insertions(+), 26 deletions(-)

diff --git a/chain/vm/gas.go b/chain/vm/gas.go
index e1f7952b9..bee65fd8f 100644
--- a/chain/vm/gas.go
+++ b/chain/vm/gas.go
@@ -200,11 +200,12 @@ func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte
 func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]abi.SealVerifyInfo) (map[address.Address][]bool, error) {
 	var gasChargeSum GasCharge
 	gasChargeSum.Name = "BatchVerifySeals"
-	count := 0
+	count := int64(0)
 	for _, svis := range inp {
-		count += len(svis)
+		count += int64(len(svis))
 	}
-	ps.chargeGas(gasChargeSum.WithExtra(count)) // TODO: this is only called by the cron actor. Should we even charge gas?
+	gasChargeSum = gasChargeSum.WithExtra(count).WithVirtual(129778623*count+716683250, 0)
+	ps.chargeGas(gasChargeSum) // TODO: this is only called by the cron actor. Should we even charge gas?
 	defer ps.chargeGas(gasOnActorExec)
 
 	return ps.under.BatchVerifySeals(inp)
diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go
index 0215b7454..b0c839f2f 100644
--- a/chain/vm/gas_v0.go
+++ b/chain/vm/gas_v0.go
@@ -99,18 +99,26 @@ func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge {
 func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) GasCharge {
 	ret := pl.sendBase
 	extra := ""
+	virtGas := int64(1069512)
 
 	if value != abi.NewTokenAmount(0) {
 		// TODO: fix this, it is comparing pointers instead of values
 		// see vv
 		ret += pl.sendTransferFunds
 	}
-	if big.Cmp(value, abi.NewTokenAmount(0)) == 0 {
+	if big.Cmp(value, abi.NewTokenAmount(0)) != 0 {
+		virtGas += 498173
+		if methodNum == builtin.MethodSend {
+			// transfer only
+			virtGas += 968198
+		}
 		extra += "t"
 	}
 	if methodNum != builtin.MethodSend {
 		ret += pl.sendInvokeMethod
 		extra += "i"
+		// running actors is cheaper becase we hand over to actors
+		virtGas += -294632
 	}
 	return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(86315, 0).WithExtra(extra)
 }
@@ -145,11 +153,19 @@ func (pl *pricelistV0) OnVerifySignature(sigType crypto.SigType, planTextSize in
 		return GasCharge{}, fmt.Errorf("cost function for signature type %d not supported", sigType)
 	}
 	sigName, _ := sigType.Name()
+	virtGas := int64(0)
+	switch sigType {
+	case crypto.SigTypeBLS:
+		virtGas = 220138570
+	case crypto.SigTypeSecp256k1:
+		virtGas = 7053730
+	}
+
 	return newGasCharge("OnVerifySignature", costFn(int64(planTextSize)), 0).
 		WithExtra(map[string]interface{}{
 			"type": sigName,
 			"size": planTextSize,
-		}), nil
+		}).WithVirtual(virtGas, 0), nil
 }
 
 // OnHashing
diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 7d759642e..b38661983 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -233,7 +233,6 @@ type stats struct {
 	timeTaken meanVar
 	gasRatio  meanVar
 
-	extra      *meanVar
 	extraCovar *covar
 }
 
@@ -242,19 +241,28 @@ type covar struct {
 	meanY float64
 	c     float64
 	n     float64
-	m2    float64
+	m2x   float64
+	m2y   float64
 }
 
 func (cov1 *covar) Covariance() float64 {
 	return cov1.c / (cov1.n - 1)
 }
 
-func (cov1 *covar) Variance() float64 {
-	return cov1.m2 / (cov1.n - 1)
+func (cov1 *covar) VarianceX() float64 {
+	return cov1.m2x / (cov1.n - 1)
 }
 
-func (v1 *covar) Stddev() float64 {
-	return math.Sqrt(v1.Variance())
+func (v1 *covar) StddevX() float64 {
+	return math.Sqrt(v1.VarianceX())
+}
+
+func (cov1 *covar) VarianceY() float64 {
+	return cov1.m2y / (cov1.n - 1)
+}
+
+func (v1 *covar) StddevY() float64 {
+	return math.Sqrt(v1.VarianceY())
 }
 
 func (cov1 *covar) AddPoint(x, y float64) {
@@ -262,11 +270,15 @@ func (cov1 *covar) AddPoint(x, y float64) {
 	dx := x - cov1.meanX
 	cov1.meanX += dx / cov1.n
 
-	dx2 := x - cov1.meanX // compute x variance using partial result for covariance
-	cov1.m2 += dx * dx2
+	dx2 := x - cov1.meanX
+	cov1.m2x += dx * dx2
 
-	cov1.meanY += (y - cov1.meanY) / cov1.n
-	cov1.c += dx * (y - cov1.meanY)
+	dy := y - cov1.meanY
+	cov1.meanY += dy / cov1.n
+	dy2 := y - cov1.meanY
+	cov1.m2y += dy * dy2
+
+	cov1.c += dx * dy
 }
 
 func (cov1 *covar) Combine(cov2 *covar) {
@@ -293,21 +305,25 @@ func (cov1 *covar) Combine(cov2 *covar) {
 
 	dx := cov1.meanX - cov2.meanX
 	out.meanX = cov1.meanX - dx*cov2.n/out.n
-	out.m2 = cov1.m2 + cov2.m2 + dx*dx*cov1.n*cov2.n/out.n
+	out.m2x = cov1.m2x + cov2.m2x + dx*dx*cov1.n*cov2.n/out.n
 
 	dy := cov1.meanY - cov2.meanY
 	out.meanY = cov1.meanY - dy*cov2.n/out.n
+	out.m2y = cov1.m2y + cov2.m2y + dy*dy*cov1.n*cov2.n/out.n
 
 	out.c = cov1.c + cov2.c + dx*dy*cov1.n*cov2.n/out.n
 	*cov1 = out
 }
 
 func (cov1 *covar) A() float64 {
-	return cov1.Covariance() / cov1.Variance()
+	return cov1.Covariance() / cov1.VarianceX()
 }
 func (cov1 *covar) B() float64 {
 	return cov1.meanY - cov1.meanX*cov1.A()
 }
+func (cov1 *covar) Coerrel() float64 {
+	return cov1.Covariance() / cov1.VarianceX() / cov1.VarianceY()
+}
 
 type meanVar struct {
 	n    float64
@@ -417,10 +433,8 @@ func tallyGasCharges(charges map[string]*stats, et types.ExecutionTrace) {
 		if eSize != nil {
 			if s.extraCovar == nil {
 				s.extraCovar = &covar{}
-				s.extra = &meanVar{}
 			}
 			s.extraCovar.AddPoint(*eSize, tt)
-			s.extra.AddPoint(*eSize)
 		}
 
 		s.timeTaken.AddPoint(tt)
@@ -550,12 +564,10 @@ var importAnalyzeCmd = &cli.Command{
 				s.timeTaken.Combine(&v.timeTaken)
 				s.gasRatio.Combine(&v.gasRatio)
 
-				if v.extra != nil {
-					if s.extra == nil {
-						s.extra = &meanVar{}
+				if v.extraCovar != nil {
+					if s.extraCovar == nil {
 						s.extraCovar = &covar{}
 					}
-					s.extra.Combine(v.extra)
 					s.extraCovar.Combine(v.extraCovar)
 				}
 			}
@@ -573,10 +585,10 @@ var importAnalyzeCmd = &cli.Command{
 			s := charges[k]
 			fmt.Printf("%s: incr by %f~%f; tt %f~%f\n", k, s.gasRatio.Mean(), s.gasRatio.Stddev(),
 				s.timeTaken.Mean(), s.timeTaken.Stddev())
-			if s.extra != nil {
-				fmt.Printf("\t covar: %f, tt = %f * extra + %f\n", s.extraCovar.Covariance(),
+			if s.extraCovar != nil {
+				fmt.Printf("\t correll: %f, tt = %f * extra + %f\n", s.extraCovar.Coerrel(),
 					s.extraCovar.A(), s.extraCovar.B())
-				fmt.Printf("\t extra: %f~%f\n", s.extra.Mean(), s.extra.Stddev())
+				fmt.Printf("\t extra: %f~%f\n", s.extraCovar.meanX, s.extraCovar.StddevX())
 			}
 		}
 

From 4dd79ec6fa2a2951f0d76dd4bbc772260b656f3d Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 14:21:48 +0200
Subject: [PATCH 36/47] Calculate correlation

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go     | 6 +++---
 cmd/lotus-bench/stats_test.go | 8 +++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index b38661983..f6a07ca05 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -321,8 +321,8 @@ func (cov1 *covar) A() float64 {
 func (cov1 *covar) B() float64 {
 	return cov1.meanY - cov1.meanX*cov1.A()
 }
-func (cov1 *covar) Coerrel() float64 {
-	return cov1.Covariance() / cov1.VarianceX() / cov1.VarianceY()
+func (cov1 *covar) Correl() float64 {
+	return cov1.Covariance() / cov1.StddevX() / cov1.StddevY()
 }
 
 type meanVar struct {
@@ -586,7 +586,7 @@ var importAnalyzeCmd = &cli.Command{
 			fmt.Printf("%s: incr by %f~%f; tt %f~%f\n", k, s.gasRatio.Mean(), s.gasRatio.Stddev(),
 				s.timeTaken.Mean(), s.timeTaken.Stddev())
 			if s.extraCovar != nil {
-				fmt.Printf("\t correll: %f, tt = %f * extra + %f\n", s.extraCovar.Coerrel(),
+				fmt.Printf("\t correll: %f, tt = %f * extra + %f\n", s.extraCovar.Correl(),
 					s.extraCovar.A(), s.extraCovar.B())
 				fmt.Printf("\t extra: %f~%f\n", s.extraCovar.meanX, s.extraCovar.StddevX())
 			}
diff --git a/cmd/lotus-bench/stats_test.go b/cmd/lotus-bench/stats_test.go
index a8cb985a1..16caf3f7e 100644
--- a/cmd/lotus-bench/stats_test.go
+++ b/cmd/lotus-bench/stats_test.go
@@ -33,13 +33,15 @@ func TestCovar(t *testing.T) {
 		maxJ := rng.Intn(1000) + 500
 		for j := 0; j < maxJ; j++ {
 			x := rng.NormFloat64()*5 + 500
-			ss[i].AddPoint(x, x*(rng.NormFloat64()/10+2)-rng.NormFloat64()-1000)
+			ss[i].AddPoint(x, x*2-1000)
 		}
-		t.Logf("covar: %f, y = %f*x+%f @%.0f", ss[i].Covariance(), ss[i].A(), ss[i].B(), ss[i].n)
+		t.Logf("corell: %f, y = %f*x+%f @%.0f", ss[i].Correl(), ss[i].A(), ss[i].B(), ss[i].n)
+		t.Logf("\txVar: %f yVar: %f covar: %f", ss[i].StddevX(), ss[i].StddevY(), ss[i].Covariance())
 	}
 	out := &covar{}
 	for i := 0; i < N; i++ {
 		out.Combine(ss[i])
-		t.Logf("combine: covar: %f, y = %f*x+%f", ss[i].Covariance(), ss[i].A(), ss[i].B())
+		t.Logf("combine: corell: %f, y = %f*x+%f", out.Correl(), out.A(), out.B())
+		t.Logf("\txVar: %f yVar: %f covar: %f", out.StddevX(), out.StddevY(), out.Covariance())
 	}
 }

From 254295b9c38b8dc63f79e8fc176003eca7dcabf0 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 14:30:58 +0200
Subject: [PATCH 37/47] Readd covariance

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index f6a07ca05..1e85d1757 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -588,7 +588,7 @@ var importAnalyzeCmd = &cli.Command{
 			if s.extraCovar != nil {
 				fmt.Printf("\t correll: %f, tt = %f * extra + %f\n", s.extraCovar.Correl(),
 					s.extraCovar.A(), s.extraCovar.B())
-				fmt.Printf("\t extra: %f~%f\n", s.extraCovar.meanX, s.extraCovar.StddevX())
+				fmt.Printf("\t covar: %f, extra: %f~%f\n", s.extraCovar.Covariance(), s.extraCovar.meanX, s.extraCovar.StddevX())
 			}
 		}
 

From 7a7e5a85ad9e7e1b105c4864d3ec36109aef0913 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 14:32:09 +0200
Subject: [PATCH 38/47] Print tt from covar

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 1e85d1757..0b2261a45 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -588,7 +588,8 @@ var importAnalyzeCmd = &cli.Command{
 			if s.extraCovar != nil {
 				fmt.Printf("\t correll: %f, tt = %f * extra + %f\n", s.extraCovar.Correl(),
 					s.extraCovar.A(), s.extraCovar.B())
-				fmt.Printf("\t covar: %f, extra: %f~%f\n", s.extraCovar.Covariance(), s.extraCovar.meanX, s.extraCovar.StddevX())
+				fmt.Printf("\t covar: %f, extra: %f~%f, tt2: %f~%f\n", s.extraCovar.Covariance(),
+					s.extraCovar.meanX, s.extraCovar.StddevX(), s.extraCovar.meanY, s.extraCovar.StddevY())
 			}
 		}
 

From 39625eada7731562e46c0d49fe55718bb1047ad4 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 14:41:14 +0200
Subject: [PATCH 39/47] Add count

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 0b2261a45..7d2455a81 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -267,9 +267,9 @@ func (v1 *covar) StddevY() float64 {
 
 func (cov1 *covar) AddPoint(x, y float64) {
 	cov1.n += 1
+
 	dx := x - cov1.meanX
 	cov1.meanX += dx / cov1.n
-
 	dx2 := x - cov1.meanX
 	cov1.m2x += dx * dx2
 
@@ -586,10 +586,11 @@ var importAnalyzeCmd = &cli.Command{
 			fmt.Printf("%s: incr by %f~%f; tt %f~%f\n", k, s.gasRatio.Mean(), s.gasRatio.Stddev(),
 				s.timeTaken.Mean(), s.timeTaken.Stddev())
 			if s.extraCovar != nil {
-				fmt.Printf("\t correll: %f, tt = %f * extra + %f\n", s.extraCovar.Correl(),
+				fmt.Printf("\t correll: %.2f, tt = %f * extra + %f\n", s.extraCovar.Correl(),
 					s.extraCovar.A(), s.extraCovar.B())
-				fmt.Printf("\t covar: %f, extra: %f~%f, tt2: %f~%f\n", s.extraCovar.Covariance(),
-					s.extraCovar.meanX, s.extraCovar.StddevX(), s.extraCovar.meanY, s.extraCovar.StddevY())
+				fmt.Printf("\t covar: %.2f, extra: %.2f~%.2f, tt2: %.2f~%.2f, count %.0f\n",
+					s.extraCovar.Covariance(), s.extraCovar.meanX, s.extraCovar.StddevX(),
+					s.extraCovar.meanY, s.extraCovar.StddevY(), s.extraCovar.n)
 			}
 		}
 

From 42696372ee244e99711fe6d969e27dd7a75f5e53 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 14:42:56 +0200
Subject: [PATCH 40/47] Progress to stderr

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 7d2455a81..f55735e59 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -538,21 +538,21 @@ var importAnalyzeCmd = &cli.Command{
 			}
 			totalTipsets++
 			jsonIn <- b
-			fmt.Printf("\rProcessed %d tipsets", totalTipsets)
+			fmt.Fprintf(os.Stderr, "\rProcessed %d tipsets", totalTipsets)
 			if err == io.EOF {
 				break
 			}
 		}
 		close(jsonIn)
-		fmt.Printf("\n")
-		fmt.Printf("Collecting results\n")
+		fmt.Fprintf(os.Stderr, "\n")
+		fmt.Fprintf(os.Stderr, "Collecting results\n")
 
 		var invocs []Invocation
 		var totalTime time.Duration
 		var keys []string
 		var charges = make(map[string]*stats)
 		for i := 0; i < nWorkers; i++ {
-			fmt.Printf("\rProcessing results from worker %d/%d", i+1, nWorkers)
+			fmt.Fprintf(os.Stderr, "\rProcessing results from worker %d/%d", i+1, nWorkers)
 			res := <-results
 			invocs = append(invocs, res.expensiveInvocs...)
 			for k, v := range res.chargeStats {
@@ -574,7 +574,7 @@ var importAnalyzeCmd = &cli.Command{
 			totalTime += res.totalTime
 		}
 
-		fmt.Printf("\nCollecting gas keys\n")
+		fmt.Fprintf(os.Stderr, "\nCollecting gas keys\n")
 		for k := range charges {
 			keys = append(keys, k)
 		}

From e9d2a3edd9b83a9f7aaf6b4e94fc355998d37ab8 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 14:45:24 +0200
Subject: [PATCH 41/47] Limit float print precision

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index f55735e59..d3a271022 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -583,10 +583,10 @@ var importAnalyzeCmd = &cli.Command{
 		sort.Strings(keys)
 		for _, k := range keys {
 			s := charges[k]
-			fmt.Printf("%s: incr by %f~%f; tt %f~%f\n", k, s.gasRatio.Mean(), s.gasRatio.Stddev(),
+			fmt.Printf("%s: incr by %.4f~%.4f; tt %.4f~%.4f\n", k, s.gasRatio.Mean(), s.gasRatio.Stddev(),
 				s.timeTaken.Mean(), s.timeTaken.Stddev())
 			if s.extraCovar != nil {
-				fmt.Printf("\t correll: %.2f, tt = %f * extra + %f\n", s.extraCovar.Correl(),
+				fmt.Printf("\t correll: %.2f, tt = %.2f * extra + %.2f\n", s.extraCovar.Correl(),
 					s.extraCovar.A(), s.extraCovar.B())
 				fmt.Printf("\t covar: %.2f, extra: %.2f~%.2f, tt2: %.2f~%.2f, count %.0f\n",
 					s.extraCovar.Covariance(), s.extraCovar.meanX, s.extraCovar.StddevX(),

From 41b775fdd48c6724cfae43cfb485b8a6c5a03173 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 19:59:34 +0200
Subject: [PATCH 42/47] Compute time spent in actors

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index d3a271022..06bdffeaa 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -600,6 +600,10 @@ var importAnalyzeCmd = &cli.Command{
 
 		fmt.Println("Total time: ", totalTime)
 		fmt.Println("Average time per epoch: ", totalTime/time.Duration(totalTipsets))
+		if actorExec, ok := charges["OnActorExec"]; ok {
+			timeInActors := actorExec.timeTaken.Mean() * actorExec.timeTaken.n
+			fmt.Printf("Avarage time per epoch in actors: %s (%.1f%%)\n", time.Duration(timeInActors)/time.Duration(totalTipsets), timeInActors/float64(totalTime)*100)
+		}
 
 		n := 30
 		if len(invocs) < n {

From 11dc902d53deb2e805e09ce9823383292a1cd5a3 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 20:12:01 +0200
Subject: [PATCH 43/47] Adjust OnMethodInvocation

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/gas_v0.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go
index b0c839f2f..37eea066a 100644
--- a/chain/vm/gas_v0.go
+++ b/chain/vm/gas_v0.go
@@ -99,7 +99,7 @@ func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge {
 func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.MethodNum) GasCharge {
 	ret := pl.sendBase
 	extra := ""
-	virtGas := int64(1069512)
+	virtGas := int64(1072944)
 
 	if value != abi.NewTokenAmount(0) {
 		// TODO: fix this, it is comparing pointers instead of values
@@ -107,10 +107,10 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M
 		ret += pl.sendTransferFunds
 	}
 	if big.Cmp(value, abi.NewTokenAmount(0)) != 0 {
-		virtGas += 498173
+		virtGas += 497495
 		if methodNum == builtin.MethodSend {
 			// transfer only
-			virtGas += 968198
+			virtGas += 973940
 		}
 		extra += "t"
 	}
@@ -118,9 +118,9 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M
 		ret += pl.sendInvokeMethod
 		extra += "i"
 		// running actors is cheaper becase we hand over to actors
-		virtGas += -294632
+		virtGas += -295779
 	}
-	return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(86315, 0).WithExtra(extra)
+	return newGasCharge("OnMethodInvocation", ret, 0).WithVirtual(virtGas, 0).WithExtra(extra)
 }
 
 // OnIpldGet returns the gas used for storing an object

From 3429941639fa20f6984125bc051cff1128e34aa8 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 20:13:22 +0200
Subject: [PATCH 44/47] Add info about OnActorExecDone

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 cmd/lotus-bench/import.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go
index 06bdffeaa..6d5f7f8ff 100644
--- a/cmd/lotus-bench/import.go
+++ b/cmd/lotus-bench/import.go
@@ -604,6 +604,10 @@ var importAnalyzeCmd = &cli.Command{
 			timeInActors := actorExec.timeTaken.Mean() * actorExec.timeTaken.n
 			fmt.Printf("Avarage time per epoch in actors: %s (%.1f%%)\n", time.Duration(timeInActors)/time.Duration(totalTipsets), timeInActors/float64(totalTime)*100)
 		}
+		if actorExecDone, ok := charges["OnActorExecDone"]; ok {
+			timeInActors := actorExecDone.timeTaken.Mean() * actorExecDone.timeTaken.n
+			fmt.Printf("Avarage time per epoch in OnActorExecDone %s (%.1f%%)\n", time.Duration(timeInActors)/time.Duration(totalTipsets), timeInActors/float64(totalTime)*100)
+		}
 
 		n := 30
 		if len(invocs) < n {

From c4158f12f120a40ac05e46c939d407e4eda9fba0 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Tue, 30 Jun 2020 23:06:36 +0200
Subject: [PATCH 45/47] Include proof number as extra in OnVerifyPost

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/gas_v0.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go
index 37eea066a..5735d0b83 100644
--- a/chain/vm/gas_v0.go
+++ b/chain/vm/gas_v0.go
@@ -188,7 +188,7 @@ func (pl *pricelistV0) OnVerifySeal(info abi.SealVerifyInfo) GasCharge {
 // OnVerifyPost
 func (pl *pricelistV0) OnVerifyPost(info abi.WindowPoStVerifyInfo) GasCharge {
 	// TODO: this needs more cost tunning, check with @lotus
-	return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0).WithVirtual(2629471704, 0)
+	return newGasCharge("OnVerifyPost", pl.verifyPostBase, 0).WithVirtual(2629471704, 0).WithExtra(len(info.ChallengedSectors))
 }
 
 // OnVerifyConsensusFault

From f3f92ce5bc03639ea92cf0a2edea223b35353adb Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Thu, 2 Jul 2020 18:42:06 +0200
Subject: [PATCH 46/47] add scratchpad to gitignore

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 6aaaf9240..fce23289f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,4 @@ build/paramfetch.sh
 bin/ipget
 bin/tmp/*
 .idea
+scratchpad

From 567b8082c31243c77cf54630796bd7d7e8606e9b Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Thu, 2 Jul 2020 18:42:42 +0200
Subject: [PATCH 47/47] Back to 10 callers

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/vm/runtime.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go
index c1581dcd9..6de8fec9b 100644
--- a/chain/vm/runtime.go
+++ b/chain/vm/runtime.go
@@ -525,7 +525,7 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) {
 
 func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError {
 	toUse := gas.Total()
-	var callers [4]uintptr
+	var callers [10]uintptr
 	cout := gruntime.Callers(2+skip, callers[:])
 
 	now := time.Now()