2020-04-02 21:18:57 +00:00
package main
import (
2020-06-26 21:38:06 +00:00
"bufio"
2020-04-02 21:18:57 +00:00
"context"
"encoding/json"
"fmt"
2020-06-23 03:08:19 +00:00
"io"
2020-04-02 21:18:57 +00:00
"io/ioutil"
2020-06-15 20:02:57 +00:00
"math"
2020-06-26 20:18:43 +00:00
"net/http"
_ "net/http/pprof"
2020-04-02 21:18:57 +00:00
"os"
2020-06-15 23:05:29 +00:00
"runtime"
2020-04-02 21:18:57 +00:00
"runtime/pprof"
2020-04-23 23:48:54 +00:00
"sort"
2020-04-02 21:18:57 +00:00
"time"
2020-09-21 20:47:03 +00:00
"github.com/cockroachdb/pebble"
"github.com/cockroachdb/pebble/bloom"
2020-04-02 21:18:57 +00:00
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
2020-07-23 02:05:11 +00:00
"github.com/filecoin-project/lotus/lib/blockstore"
2020-05-15 20:01:45 +00:00
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
2020-10-10 19:57:19 +00:00
metricsprometheus "github.com/ipfs/go-metrics-prometheus"
2020-09-16 18:10:00 +00:00
"github.com/ipld/go-car"
2020-10-10 19:57:19 +00:00
"github.com/prometheus/client_golang/prometheus/promhttp"
2020-07-23 02:05:11 +00:00
2020-09-07 03:49:10 +00:00
"github.com/filecoin-project/go-state-types/abi"
2020-08-17 13:26:18 +00:00
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
2020-04-02 21:18:57 +00:00
2020-09-18 13:17:13 +00:00
bdg "github.com/dgraph-io/badger/v2"
2020-04-02 21:18:57 +00:00
"github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger2"
2020-10-10 18:49:15 +00:00
measure "github.com/ipfs/go-ds-measure"
2020-09-21 20:47:03 +00:00
pebbleds "github.com/ipfs/go-ds-pebble"
2020-09-18 13:17:13 +00:00
2020-06-02 18:12:53 +00:00
"github.com/urfave/cli/v2"
2020-07-23 02:05:11 +00:00
"golang.org/x/xerrors"
2020-04-02 21:18:57 +00:00
)
type TipSetExec struct {
TipSet types . TipSetKey
Trace [ ] * api . InvocResult
Duration time . Duration
}
var importBenchCmd = & cli . Command {
Name : "import" ,
Usage : "benchmark chain import and validation" ,
2020-04-23 23:48:54 +00:00
Subcommands : [ ] * cli . Command {
importAnalyzeCmd ,
} ,
2020-04-02 21:18:57 +00:00
Flags : [ ] cli . Flag {
& cli . Int64Flag {
Name : "height" ,
Usage : "halt validation after given height" ,
} ,
2020-06-15 23:05:29 +00:00
& cli . IntFlag {
Name : "batch-seal-verify-threads" ,
Usage : "set the parallelism factor for batch seal verification" ,
Value : runtime . NumCPU ( ) ,
} ,
2020-09-15 18:55:54 +00:00
& cli . StringFlag {
Name : "repodir" ,
Usage : "set the repo directory for the lotus bench run (defaults to /tmp)" ,
} ,
2020-09-16 14:34:54 +00:00
& cli . StringFlag {
Name : "syscall-cache" ,
Usage : "read and write syscall results from datastore" ,
} ,
& cli . BoolFlag {
Name : "export-traces" ,
Usage : "should we export execution traces" ,
Value : true ,
} ,
2020-09-16 18:10:00 +00:00
& cli . BoolFlag {
Name : "no-import" ,
Usage : "should we import the chain? if set to true chain has to be previously imported" ,
} ,
2020-09-19 18:27:24 +00:00
& cli . BoolFlag {
Name : "global-profile" ,
Value : true ,
} ,
2020-09-19 17:11:37 +00:00
& cli . Int64Flag {
Name : "start-at" ,
} ,
2020-09-21 22:54:11 +00:00
& cli . BoolFlag {
Name : "only-import" ,
} ,
2020-10-10 18:49:15 +00:00
& cli . BoolFlag {
Name : "use-pebble" ,
} ,
2020-04-02 21:18:57 +00:00
} ,
Action : func ( cctx * cli . Context ) error {
2020-10-22 12:57:55 +00:00
metricsprometheus . Inject ( ) //nolint:errcheck
2020-06-15 23:05:29 +00:00
vm . BatchSealVerifyParallelism = cctx . Int ( "batch-seal-verify-threads" )
2020-04-02 21:18:57 +00:00
if ! cctx . Args ( ) . Present ( ) {
fmt . Println ( "must pass car file of chain to benchmark importing" )
return nil
}
cfi , err := os . Open ( cctx . Args ( ) . First ( ) )
if err != nil {
return err
}
2020-05-27 20:53:20 +00:00
defer cfi . Close ( ) //nolint:errcheck // read only file
2020-04-02 21:18:57 +00:00
2020-09-18 11:55:37 +00:00
go func ( ) {
2020-10-10 19:57:19 +00:00
http . Handle ( "/debug/metrics/prometheus" , promhttp . Handler ( ) )
2020-09-18 11:55:37 +00:00
http . ListenAndServe ( "localhost:6060" , nil ) //nolint:errcheck
} ( )
2020-09-15 18:55:54 +00:00
var tdir string
if rdir := cctx . String ( "repodir" ) ; rdir != "" {
tdir = rdir
} else {
tmp , err := ioutil . TempDir ( "" , "lotus-import-bench" )
if err != nil {
return err
}
tdir = tmp
2020-04-02 21:18:57 +00:00
}
2020-09-21 22:54:11 +00:00
bdgOpt := badger . DefaultOptions
bdgOpt . GcInterval = 0
bdgOpt . Options = bdg . DefaultOptions ( "" )
bdgOpt . Options . SyncWrites = false
bdgOpt . Options . Truncate = true
bdgOpt . Options . DetectConflicts = false
2020-09-21 22:46:31 +00:00
var bds datastore . Batching
2020-10-10 18:49:15 +00:00
if cctx . Bool ( "use-pebble" ) {
2020-09-21 22:46:31 +00:00
cache := 512
bds , err = pebbleds . NewDatastore ( tdir , & pebble . Options {
// Pebble has a single combined cache area and the write
// buffers are taken from this too. Assign all available
// memory allowance for cache.
Cache : pebble . NewCache ( int64 ( cache * 1024 * 1024 ) ) ,
// The size of memory table(as well as the write buffer).
// Note, there may have more than two memory tables in the system.
// MemTableStopWritesThreshold can be configured to avoid the memory abuse.
MemTableSize : cache * 1024 * 1024 / 4 ,
// The default compaction concurrency(1 thread),
// Here use all available CPUs for faster compaction.
MaxConcurrentCompactions : runtime . NumCPU ( ) ,
// Per-level options. Options for at least one level must be specified. The
// options for the last level are used for all subsequent levels.
Levels : [ ] pebble . LevelOptions {
2020-09-21 22:57:37 +00:00
{ TargetFileSize : 16 * 1024 * 1024 , FilterPolicy : bloom . FilterPolicy ( 10 ) , Compression : pebble . NoCompression } ,
2020-09-21 22:46:31 +00:00
} ,
Logger : log ,
} )
} else {
bds , err = badger . NewDatastore ( tdir , & bdgOpt )
}
2020-04-02 21:18:57 +00:00
if err != nil {
return err
}
2020-09-23 16:44:41 +00:00
defer bds . Close ( ) //nolint:errcheck
2020-09-18 11:39:38 +00:00
2020-10-10 18:49:15 +00:00
bds = measure . New ( "dsbench" , bds )
2020-04-02 21:18:57 +00:00
bs := blockstore . NewBlockstore ( bds )
2020-09-19 18:49:40 +00:00
cacheOpts := blockstore . DefaultCacheOpts ( )
cacheOpts . HasBloomFilterSize = 0
2020-09-19 18:50:06 +00:00
cbs , err := blockstore . CachedBlockstore ( context . TODO ( ) , bs , cacheOpts )
2020-05-16 18:31:14 +00:00
if err != nil {
return err
}
bs = cbs
2020-04-02 21:18:57 +00:00
ds := datastore . NewMapDatastore ( )
2020-09-16 14:34:54 +00:00
var verifier ffiwrapper . Verifier = ffiwrapper . ProofVerifier
if cctx . IsSet ( "syscall-cache" ) {
scds , err := badger . NewDatastore ( cctx . String ( "syscall-cache" ) , & bdgOpt )
if err != nil {
return xerrors . Errorf ( "opening syscall-cache datastore: %w" , err )
}
2020-09-23 16:44:41 +00:00
defer scds . Close ( ) //nolint:errcheck
2020-09-18 11:54:20 +00:00
2020-09-16 14:34:54 +00:00
verifier = & cachingVerifier {
ds : scds ,
backend : verifier ,
}
}
2020-09-18 11:39:38 +00:00
if cctx . Bool ( "only-gc" ) {
return nil
}
2020-09-16 14:34:54 +00:00
2020-11-03 22:04:44 +00:00
cs := store . NewChainStore ( bs , bs , ds , vm . Syscalls ( verifier ) , nil )
2020-04-02 21:18:57 +00:00
stm := stmgr . NewStateManager ( cs )
2020-09-19 18:27:24 +00:00
if cctx . Bool ( "global-profile" ) {
prof , err := os . Create ( "import-bench.prof" )
if err != nil {
return err
}
defer prof . Close ( ) //nolint:errcheck
2020-04-02 21:18:57 +00:00
2020-09-19 18:27:24 +00:00
if err := pprof . StartCPUProfile ( prof ) ; err != nil {
return err
}
2020-04-02 21:18:57 +00:00
}
2020-09-16 18:10:00 +00:00
var head * types . TipSet
if ! cctx . Bool ( "no-import" ) {
head , err = cs . Import ( cfi )
if err != nil {
return err
}
} else {
cr , err := car . NewCarReader ( cfi )
if err != nil {
return err
}
head , err = cs . LoadTipSet ( types . NewTipSetKey ( cr . Header . Roots ... ) )
if err != nil {
return err
}
2020-04-02 21:18:57 +00:00
}
2020-09-21 22:54:11 +00:00
if cctx . Bool ( "only-import" ) {
return nil
}
2020-08-04 04:45:02 +00:00
gb , err := cs . GetTipsetByHeight ( context . TODO ( ) , 0 , head , true )
if err != nil {
return err
}
err = cs . SetGenesis ( gb . Blocks ( ) [ 0 ] )
if err != nil {
return err
}
2020-09-19 17:11:37 +00:00
startEpoch := abi . ChainEpoch ( 1 )
if cctx . IsSet ( "start-at" ) {
startEpoch = abi . ChainEpoch ( cctx . Int64 ( "start-at" ) )
start , err := cs . GetTipsetByHeight ( context . TODO ( ) , abi . ChainEpoch ( cctx . Int64 ( "start-at" ) ) , head , true )
if err != nil {
return err
}
err = cs . SetHead ( start )
if err != nil {
return err
}
}
2020-04-02 21:18:57 +00:00
if h := cctx . Int64 ( "height" ) ; h != 0 {
2020-05-05 17:06:05 +00:00
tsh , err := cs . GetTipsetByHeight ( context . TODO ( ) , abi . ChainEpoch ( h ) , head , true )
2020-04-02 21:18:57 +00:00
if err != nil {
return err
}
head = tsh
}
ts := head
tschain := [ ] * types . TipSet { ts }
2020-09-19 17:11:37 +00:00
for ts . Height ( ) > startEpoch {
2020-04-02 21:18:57 +00:00
next , err := cs . LoadTipSet ( ts . Parents ( ) )
if err != nil {
return err
}
tschain = append ( tschain , next )
ts = next
}
2020-09-16 14:34:54 +00:00
var enc * json . Encoder
if cctx . Bool ( "export-traces" ) {
ibj , err := os . Create ( "import-bench.json" )
if err != nil {
return err
}
defer ibj . Close ( ) //nolint:errcheck
2020-06-23 03:08:19 +00:00
2020-09-16 14:34:54 +00:00
enc = json . NewEncoder ( ibj )
}
2020-06-23 03:08:19 +00:00
2020-09-19 17:11:37 +00:00
for i := len ( tschain ) - 1 ; i >= 1 ; i -- {
2020-04-02 21:18:57 +00:00
cur := tschain [ i ]
start := time . Now ( )
2020-09-19 17:11:37 +00:00
log . Infof ( "computing state (height: %d, ts=%s)" , cur . Height ( ) , cur . Cids ( ) )
2020-04-02 21:18:57 +00:00
st , trace , err := stm . ExecutionTrace ( context . TODO ( ) , cur )
if err != nil {
return err
}
2020-09-19 17:11:37 +00:00
tse := & TipSetExec {
2020-09-16 18:10:00 +00:00
TipSet : cur . Key ( ) ,
Trace : trace ,
Duration : time . Since ( start ) ,
}
2020-09-16 14:34:54 +00:00
if enc != nil {
2020-09-19 17:11:37 +00:00
stripCallers ( tse . Trace )
2020-06-23 03:08:19 +00:00
2020-09-19 17:11:37 +00:00
if err := enc . Encode ( tse ) ; err != nil {
2020-09-16 14:34:54 +00:00
return xerrors . Errorf ( "failed to write out tipsetexec: %w" , err )
}
2020-06-23 03:08:19 +00:00
}
2020-09-19 17:11:37 +00:00
if tschain [ i - 1 ] . ParentState ( ) != st {
stripCallers ( tse . Trace )
lastTrace := tse . Trace
d , err := json . MarshalIndent ( lastTrace , "" , " " )
if err != nil {
panic ( err )
}
fmt . Println ( "TRACE" )
fmt . Println ( string ( d ) )
2020-09-23 16:44:41 +00:00
//fmt.Println(statediff.Diff(context.Background(), bs, tschain[i-1].ParentState(), st, statediff.ExpandActors))
2020-09-19 17:11:37 +00:00
return xerrors . Errorf ( "tipset chain had state mismatch at height %d (%s != %s)" , cur . Height ( ) , cur . ParentState ( ) , st )
}
2020-04-02 21:18:57 +00:00
}
pprof . StopCPUProfile ( )
2020-10-14 04:24:56 +00:00
if true {
resp , err := http . Get ( "http://localhost:6060/debug/metrics/prometheus" )
if err != nil {
return err
}
metricsfi , err := os . Create ( "import-bench.metrics" )
if err != nil {
return err
}
2020-10-22 12:57:55 +00:00
io . Copy ( metricsfi , resp . Body ) //nolint:errcheck
metricsfi . Close ( ) //nolint:errcheck
2020-10-14 04:24:56 +00:00
}
2020-04-02 21:18:57 +00:00
return nil
} ,
}
2020-04-23 23:48:54 +00:00
2020-06-25 14:46:50 +00:00
func walkExecutionTrace ( et * types . ExecutionTrace ) {
for _ , gc := range et . GasCharges {
gc . Callers = nil
}
for _ , sub := range et . Subcalls {
2020-06-25 14:58:55 +00:00
walkExecutionTrace ( & sub ) //nolint:scopelint,gosec
2020-06-25 14:46:50 +00:00
}
}
func stripCallers ( trace [ ] * api . InvocResult ) {
for _ , t := range trace {
walkExecutionTrace ( & t . ExecutionTrace )
}
}
2020-04-23 23:48:54 +00:00
type Invocation struct {
TipSet types . TipSetKey
Invoc * api . InvocResult
}
2020-06-15 20:02:57 +00:00
const GasPerNs = 10
func countGasCosts ( et * types . ExecutionTrace ) ( int64 , int64 ) {
var cgas , vgas int64
for _ , gc := range et . GasCharges {
cgas += gc . ComputeGas
vgas += gc . VirtualComputeGas
}
for _ , sub := range et . Subcalls {
2020-08-20 04:49:10 +00:00
c , v := countGasCosts ( & sub ) //nolint
2020-06-15 20:02:57 +00:00
cgas += c
vgas += v
}
return cgas , vgas
}
2020-06-26 21:08:06 +00:00
type stats struct {
2020-06-29 23:31:09 +00:00
timeTaken meanVar
gasRatio meanVar
extraCovar * covar
}
type covar struct {
meanX float64
meanY float64
c float64
n float64
2020-06-30 12:09:03 +00:00
m2x float64
m2y float64
2020-06-26 21:08:06 +00:00
}
2020-06-29 23:31:09 +00:00
func ( cov1 * covar ) Covariance ( ) float64 {
return cov1 . c / ( cov1 . n - 1 )
}
2020-06-30 12:09:03 +00:00
func ( cov1 * covar ) VarianceX ( ) float64 {
return cov1 . m2x / ( cov1 . n - 1 )
2020-06-30 00:11:54 +00:00
}
2020-08-20 04:49:10 +00:00
func ( cov1 * covar ) StddevX ( ) float64 {
return math . Sqrt ( cov1 . VarianceX ( ) )
2020-06-30 12:09:03 +00:00
}
func ( cov1 * covar ) VarianceY ( ) float64 {
return cov1 . m2y / ( cov1 . n - 1 )
}
2020-08-20 04:49:10 +00:00
func ( cov1 * covar ) StddevY ( ) float64 {
return math . Sqrt ( cov1 . VarianceY ( ) )
2020-06-30 00:11:54 +00:00
}
2020-06-29 23:31:09 +00:00
func ( cov1 * covar ) AddPoint ( x , y float64 ) {
2020-08-20 04:49:10 +00:00
cov1 . n ++
2020-06-30 12:41:14 +00:00
2020-06-29 23:31:09 +00:00
dx := x - cov1 . meanX
cov1 . meanX += dx / cov1 . n
2020-06-30 12:09:03 +00:00
dx2 := x - cov1 . meanX
cov1 . m2x += dx * dx2
dy := y - cov1 . meanY
cov1 . meanY += dy / cov1 . n
dy2 := y - cov1 . meanY
cov1 . m2y += dy * dy2
2020-06-29 23:31:09 +00:00
2020-06-30 12:09:03 +00:00
cov1 . c += dx * dy
2020-06-29 23:31:09 +00:00
}
func ( cov1 * covar ) Combine ( cov2 * covar ) {
if cov1 . n == 0 {
* cov1 = * cov2
return
}
if cov2 . n == 0 {
return
}
if cov1 . n == 1 {
cpy := * cov2
cpy . AddPoint ( cov2 . meanX , cov2 . meanY )
* cov1 = cpy
return
}
if cov2 . n == 1 {
cov1 . AddPoint ( cov2 . meanX , cov2 . meanY )
}
2020-06-30 00:11:54 +00:00
2020-06-29 23:31:09 +00:00
out := covar { }
out . n = cov1 . n + cov2 . n
dx := cov1 . meanX - cov2 . meanX
2020-06-30 00:11:54 +00:00
out . meanX = cov1 . meanX - dx * cov2 . n / out . n
2020-06-30 12:09:03 +00:00
out . m2x = cov1 . m2x + cov2 . m2x + dx * dx * cov1 . n * cov2 . n / out . n
2020-06-29 23:31:09 +00:00
dy := cov1 . meanY - cov2 . meanY
2020-06-30 00:56:58 +00:00
out . meanY = cov1 . meanY - dy * cov2 . n / out . n
2020-06-30 12:09:03 +00:00
out . m2y = cov1 . m2y + cov2 . m2y + dy * dy * cov1 . n * cov2 . n / out . n
2020-06-29 23:31:09 +00:00
out . c = cov1 . c + cov2 . c + dx * dy * cov1 . n * cov2 . n / out . n
* cov1 = out
}
2020-06-30 00:11:54 +00:00
func ( cov1 * covar ) A ( ) float64 {
2020-06-30 12:09:03 +00:00
return cov1 . Covariance ( ) / cov1 . VarianceX ( )
2020-06-30 00:11:54 +00:00
}
func ( cov1 * covar ) B ( ) float64 {
return cov1 . meanY - cov1 . meanX * cov1 . A ( )
}
2020-06-30 12:21:48 +00:00
func ( cov1 * covar ) Correl ( ) float64 {
return cov1 . Covariance ( ) / cov1 . StddevX ( ) / cov1 . StddevY ( )
2020-06-30 12:09:03 +00:00
}
2020-06-30 00:11:54 +00:00
2020-06-29 23:31:09 +00:00
type meanVar struct {
n float64
mean float64
m2 float64
}
func ( v1 * meanVar ) AddPoint ( value float64 ) {
2020-06-29 22:31:05 +00:00
// based on https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
2020-08-20 04:49:10 +00:00
v1 . n ++
2020-06-29 23:31:09 +00:00
delta := value - v1 . mean
v1 . mean += delta / v1 . n
delta2 := value - v1 . mean
v1 . m2 += delta * delta2
2020-06-26 21:08:06 +00:00
}
2020-06-29 23:31:09 +00:00
func ( v1 * meanVar ) Variance ( ) float64 {
return v1 . m2 / ( v1 . n - 1 )
}
func ( v1 * meanVar ) Mean ( ) float64 {
return v1 . mean
}
func ( v1 * meanVar ) Stddev ( ) float64 {
return math . Sqrt ( v1 . Variance ( ) )
2020-06-26 21:08:06 +00:00
}
2020-06-29 23:31:09 +00:00
func ( v1 * meanVar ) Combine ( v2 * meanVar ) {
if v1 . n == 0 {
* v1 = * v2
2020-06-27 11:59:38 +00:00
return
2020-06-26 21:08:06 +00:00
}
2020-06-29 23:31:09 +00:00
if v2 . n == 0 {
2020-06-27 11:55:29 +00:00
return
}
2020-06-29 23:31:09 +00:00
if v1 . n == 1 {
cpy := * v2
cpy . AddPoint ( v1 . mean )
* v1 = cpy
2020-06-27 11:59:38 +00:00
return
}
2020-06-29 23:31:09 +00:00
if v2 . n == 1 {
v1 . AddPoint ( v2 . mean )
2020-06-27 11:59:38 +00:00
return
}
2020-06-29 23:31:09 +00:00
newCount := v1 . n + v2 . n
delta := v2 . mean - v1 . mean
meanDelta := delta * v2 . n / newCount
m2 := v1 . m2 + v2 . m2 + delta * meanDelta * v1 . n
v1 . n = newCount
v1 . mean += meanDelta
v1 . m2 = m2
2020-06-26 21:08:06 +00:00
}
2020-06-30 00:56:58 +00:00
func getExtras ( ex interface { } ) ( * string , * float64 ) {
if t , ok := ex . ( string ) ; ok {
return & t , nil
}
if size , ok := ex . ( float64 ) ; ok {
return nil , & size
}
if exMap , ok := ex . ( map [ string ] interface { } ) ; ok {
t , tok := exMap [ "type" ] . ( string )
size , sok := exMap [ "size" ] . ( float64 )
if tok && sok {
return & t , & size
}
if tok {
return & t , nil
}
if sok {
return nil , & size
}
return nil , nil
}
return nil , nil
}
2020-06-26 23:09:35 +00:00
func tallyGasCharges ( charges map [ string ] * stats , et types . ExecutionTrace ) {
2020-06-30 00:11:54 +00:00
for i , gc := range et . GasCharges {
2020-06-29 17:19:02 +00:00
name := gc . Name
2020-07-15 19:40:49 +00:00
if name == "OnIpldGetEnd" {
2020-06-30 00:11:54 +00:00
continue
}
tt := float64 ( gc . TimeTaken . Nanoseconds ( ) )
2020-07-28 19:00:19 +00:00
if name == "OnVerifyPost" && tt > 2e9 {
2020-07-28 19:33:49 +00:00
log . Warnf ( "Skipping abnormally long OnVerifyPost: %fs" , tt / 1e9 )
2020-07-28 19:00:19 +00:00
// discard initial very long OnVerifyPost
continue
}
eType , eSize := getExtras ( gc . Extra )
2020-06-30 00:11:54 +00:00
if name == "OnIpldGet" {
2020-07-15 19:40:49 +00:00
next := & types . GasTrace { }
if i + 1 < len ( et . GasCharges ) {
next = et . GasCharges [ i + 1 ]
2020-06-30 00:11:54 +00:00
}
2020-07-15 19:40:49 +00:00
if next . Name != "OnIpldGetEnd" {
log . Warn ( "OnIpldGet without OnIpldGetEnd" )
2020-07-28 19:00:19 +00:00
} else {
_ , size := getExtras ( next . Extra )
eSize = size
2020-07-15 19:40:49 +00:00
}
2020-06-30 00:11:54 +00:00
}
2020-06-30 00:56:58 +00:00
if eType != nil {
name += "-" + * eType
2020-06-29 17:19:02 +00:00
}
2020-07-28 19:00:19 +00:00
compGas := gc . ComputeGas
if compGas == 0 {
compGas = gc . VirtualComputeGas
}
2020-06-26 23:49:01 +00:00
if compGas == 0 {
compGas = 1
}
2020-06-29 17:19:02 +00:00
s := charges [ name ]
2020-06-26 21:08:06 +00:00
if s == nil {
s = new ( stats )
2020-06-29 17:19:02 +00:00
charges [ name ] = s
2020-06-26 21:08:06 +00:00
}
2020-06-30 00:56:58 +00:00
if eSize != nil {
2020-06-30 00:11:54 +00:00
if s . extraCovar == nil {
s . extraCovar = & covar { }
}
2020-06-30 00:56:58 +00:00
s . extraCovar . AddPoint ( * eSize , tt )
2020-06-30 00:11:54 +00:00
}
s . timeTaken . AddPoint ( tt )
ratio := tt / float64 ( compGas ) * GasPerNs
2020-06-29 23:31:09 +00:00
s . gasRatio . AddPoint ( ratio )
2020-06-15 23:05:29 +00:00
}
2020-06-26 23:18:06 +00:00
for _ , sub := range et . Subcalls {
tallyGasCharges ( charges , sub )
}
2020-06-15 23:05:29 +00:00
}
2020-04-23 23:48:54 +00:00
var importAnalyzeCmd = & cli . Command {
Name : "analyze" ,
Action : func ( cctx * cli . Context ) error {
if ! cctx . Args ( ) . Present ( ) {
fmt . Println ( "must pass bench file to analyze" )
return nil
}
2020-06-26 20:18:43 +00:00
go func ( ) {
2020-08-20 04:49:10 +00:00
http . ListenAndServe ( "localhost:6060" , nil ) //nolint:errcheck
2020-06-26 20:18:43 +00:00
} ( )
2020-04-23 23:48:54 +00:00
fi , err := os . Open ( cctx . Args ( ) . First ( ) )
if err != nil {
return err
}
2020-07-23 10:21:13 +00:00
defer fi . Close ( ) //nolint:errcheck
2020-04-23 23:48:54 +00:00
2020-06-26 19:54:20 +00:00
const nWorkers = 16
2020-06-26 21:38:06 +00:00
jsonIn := make ( chan [ ] byte , 2 * nWorkers )
2020-06-26 19:54:20 +00:00
type result struct {
totalTime time . Duration
2020-06-26 21:08:06 +00:00
chargeStats map [ string ] * stats
2020-06-26 19:54:20 +00:00
expensiveInvocs [ ] Invocation
}
results := make ( chan result , nWorkers )
for i := 0 ; i < nWorkers ; i ++ {
go func ( ) {
2020-06-26 21:08:06 +00:00
chargeStats := make ( map [ string ] * stats )
2020-06-26 19:54:20 +00:00
var totalTime time . Duration
2020-06-26 21:38:06 +00:00
const invocsKeep = 32
var expensiveInvocs = make ( [ ] Invocation , 0 , 8 * invocsKeep )
2020-06-26 19:54:20 +00:00
var leastExpensiveInvoc = time . Duration ( 0 )
for {
2020-06-26 21:38:06 +00:00
b , ok := <- jsonIn
2020-06-26 19:54:20 +00:00
if ! ok {
results <- result {
totalTime : totalTime ,
2020-06-26 21:08:06 +00:00
chargeStats : chargeStats ,
2020-06-26 19:54:20 +00:00
expensiveInvocs : expensiveInvocs ,
}
return
}
2020-06-26 20:18:43 +00:00
2020-06-26 23:39:41 +00:00
var tse TipSetExec
err := json . Unmarshal ( b , & tse )
if err != nil {
log . Warnf ( "error unmarshaling tipset: %+v" , err )
continue
}
2020-06-26 19:54:20 +00:00
totalTime += tse . Duration
for _ , inv := range tse . Trace {
if inv . Duration > leastExpensiveInvoc {
expensiveInvocs = append ( expensiveInvocs , Invocation {
TipSet : tse . TipSet ,
Invoc : inv ,
} )
}
2020-06-28 16:47:16 +00:00
tallyGasCharges ( chargeStats , inv . ExecutionTrace )
2020-06-26 19:54:20 +00:00
}
2020-06-26 21:38:06 +00:00
if len ( expensiveInvocs ) > 4 * invocsKeep {
sort . Slice ( expensiveInvocs , func ( i , j int ) bool {
return expensiveInvocs [ i ] . Invoc . Duration > expensiveInvocs [ j ] . Invoc . Duration
} )
2020-06-26 19:54:20 +00:00
leastExpensiveInvoc = expensiveInvocs [ len ( expensiveInvocs ) - 1 ] . Invoc . Duration
2020-06-26 19:57:57 +00:00
n := 30
if len ( expensiveInvocs ) < n {
n = len ( expensiveInvocs )
}
expensiveInvocs = expensiveInvocs [ : n ]
2020-06-26 19:54:20 +00:00
}
}
} ( )
}
var totalTipsets int64
2020-06-26 21:38:06 +00:00
reader := bufio . NewReader ( fi )
2020-06-23 03:08:19 +00:00
for {
2020-06-26 21:38:06 +00:00
b , err := reader . ReadBytes ( '\n' )
if err != nil && err != io . EOF {
if e , ok := err . ( * json . SyntaxError ) ; ok {
log . Warnf ( "syntax error at byte offset %d" , e . Offset )
2020-06-23 03:08:19 +00:00
}
2020-06-26 21:38:06 +00:00
return err
2020-06-23 03:08:19 +00:00
}
2020-06-26 19:54:20 +00:00
totalTipsets ++
2020-06-26 21:38:06 +00:00
jsonIn <- b
2020-06-30 12:42:56 +00:00
fmt . Fprintf ( os . Stderr , "\rProcessed %d tipsets" , totalTipsets )
2020-06-26 21:38:06 +00:00
if err == io . EOF {
break
2020-06-26 19:54:20 +00:00
}
2020-04-23 23:48:54 +00:00
}
2020-06-26 21:38:06 +00:00
close ( jsonIn )
2020-06-30 12:42:56 +00:00
fmt . Fprintf ( os . Stderr , "\n" )
fmt . Fprintf ( os . Stderr , "Collecting results\n" )
2020-06-15 20:02:57 +00:00
2020-04-23 23:48:54 +00:00
var invocs [ ] Invocation
var totalTime time . Duration
2020-06-26 19:54:20 +00:00
var keys [ ] string
2020-06-26 21:08:06 +00:00
var charges = make ( map [ string ] * stats )
2020-06-26 19:54:20 +00:00
for i := 0 ; i < nWorkers ; i ++ {
2020-06-30 12:42:56 +00:00
fmt . Fprintf ( os . Stderr , "\rProcessing results from worker %d/%d" , i + 1 , nWorkers )
2020-06-26 19:54:20 +00:00
res := <- results
invocs = append ( invocs , res . expensiveInvocs ... )
2020-06-26 21:08:06 +00:00
for k , v := range res . chargeStats {
s := charges [ k ]
if s == nil {
s = new ( stats )
charges [ k ] = s
}
2020-06-29 23:31:09 +00:00
s . timeTaken . Combine ( & v . timeTaken )
s . gasRatio . Combine ( & v . gasRatio )
2020-06-30 00:11:54 +00:00
2020-06-30 12:09:03 +00:00
if v . extraCovar != nil {
if s . extraCovar == nil {
2020-06-30 00:11:54 +00:00
s . extraCovar = & covar { }
}
s . extraCovar . Combine ( v . extraCovar )
}
2020-04-23 23:48:54 +00:00
}
2020-06-26 19:54:20 +00:00
totalTime += res . totalTime
2020-04-23 23:48:54 +00:00
}
2020-06-30 12:42:56 +00:00
fmt . Fprintf ( os . Stderr , "\nCollecting gas keys\n" )
2020-06-26 21:08:06 +00:00
for k := range charges {
2020-06-15 20:02:57 +00:00
keys = append ( keys , k )
}
fmt . Println ( "Gas Price Deltas" )
sort . Strings ( keys )
for _ , k := range keys {
2020-06-26 21:08:06 +00:00
s := charges [ k ]
2020-06-30 12:45:24 +00:00
fmt . Printf ( "%s: incr by %.4f~%.4f; tt %.4f~%.4f\n" , k , s . gasRatio . Mean ( ) , s . gasRatio . Stddev ( ) ,
2020-06-30 00:11:54 +00:00
s . timeTaken . Mean ( ) , s . timeTaken . Stddev ( ) )
2020-06-30 12:09:03 +00:00
if s . extraCovar != nil {
2020-06-30 12:45:24 +00:00
fmt . Printf ( "\t correll: %.2f, tt = %.2f * extra + %.2f\n" , s . extraCovar . Correl ( ) ,
2020-06-30 00:11:54 +00:00
s . extraCovar . A ( ) , s . extraCovar . B ( ) )
2020-06-30 12:41:14 +00:00
fmt . Printf ( "\t covar: %.2f, extra: %.2f~%.2f, tt2: %.2f~%.2f, count %.0f\n" ,
s . extraCovar . Covariance ( ) , s . extraCovar . meanX , s . extraCovar . StddevX ( ) ,
s . extraCovar . meanY , s . extraCovar . StddevY ( ) , s . extraCovar . n )
2020-06-30 00:11:54 +00:00
}
2020-06-15 20:02:57 +00:00
}
2020-04-23 23:48:54 +00:00
sort . Slice ( invocs , func ( i , j int ) bool {
return invocs [ i ] . Invoc . Duration > invocs [ j ] . Invoc . Duration
} )
fmt . Println ( "Total time: " , totalTime )
2020-06-26 19:54:20 +00:00
fmt . Println ( "Average time per epoch: " , totalTime / time . Duration ( totalTipsets ) )
2020-06-30 17:59:34 +00:00
if actorExec , ok := charges [ "OnActorExec" ] ; ok {
timeInActors := actorExec . timeTaken . Mean ( ) * actorExec . timeTaken . n
fmt . Printf ( "Avarage time per epoch in actors: %s (%.1f%%)\n" , time . Duration ( timeInActors ) / time . Duration ( totalTipsets ) , timeInActors / float64 ( totalTime ) * 100 )
}
2020-07-03 13:13:23 +00:00
if actorExecDone , ok := charges [ "OnMethodInvocationDone" ] ; ok {
2020-06-30 18:13:22 +00:00
timeInActors := actorExecDone . timeTaken . Mean ( ) * actorExecDone . timeTaken . n
fmt . Printf ( "Avarage time per epoch in OnActorExecDone %s (%.1f%%)\n" , time . Duration ( timeInActors ) / time . Duration ( totalTipsets ) , timeInActors / float64 ( totalTime ) * 100 )
}
2020-04-23 23:48:54 +00:00
n := 30
2020-06-26 18:42:29 +00:00
if len ( invocs ) < n {
n = len ( invocs )
}
2020-04-23 23:48:54 +00:00
fmt . Printf ( "Top %d most expensive calls:\n" , n )
for i := 0 ; i < n ; i ++ {
inv := invocs [ i ] . Invoc
fmt . Printf ( "%s: %s %s %d %s\n" , inv . Duration , inv . Msg . From , inv . Msg . To , inv . Msg . Method , invocs [ i ] . TipSet )
}
return nil
} ,
}