cerc-io/lotus
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #4302 from filecoin-project/feat/import-bench-ds-measure

Browse Source 
add measure datastore wrapper around bench chain datastore
This commit is contained in:
Łukasz Magiera 2020-10-22 15:16:31 +02:00 committed by GitHub
commit 93227bfd4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 33 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
cmd/lotus-bench/import.go
View File

@ -26,7 +26,9 @@ import (
"github.com/filecoin-project/lotus/lib/blockstore"
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
metricsprometheus "github.com/ipfs/go-metrics-prometheus"
"github.com/ipld/go-car"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
@ -34,6 +36,7 @@ import (
bdg "github.com/dgraph-io/badger/v2"
"github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger2"
measure "github.com/ipfs/go-ds-measure"
pebbleds "github.com/ipfs/go-ds-pebble"
"github.com/urfave/cli/v2"
@ -89,8 +92,12 @@ var importBenchCmd = &cli.Command{
&cli.BoolFlag{
Name: "only-import",
},
&cli.BoolFlag{
Name: "use-pebble",
},
},
Action: func(cctx *cli.Context) error {
metricsprometheus.Inject() //nolint:errcheck
vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads")
if !cctx.Args().Present() {
fmt.Println("must pass car file of chain to benchmark importing")
@ -104,6 +111,7 @@ var importBenchCmd = &cli.Command{
defer cfi.Close() //nolint:errcheck // read only file
go func() {
http.Handle("/debug/metrics/prometheus", promhttp.Handler())
http.ListenAndServe("localhost:6060", nil) //nolint:errcheck
}()
@ -126,7 +134,7 @@ var importBenchCmd = &cli.Command{
bdgOpt.Options.DetectConflicts = false
var bds datastore.Batching
if false {
if cctx.Bool("use-pebble") {
cache := 512
bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{
// Pebble has a single combined cache area and the write
@ -155,6 +163,8 @@ var importBenchCmd = &cli.Command{
}
defer bds.Close() //nolint:errcheck
bds = measure.New("dsbench", bds)
bs := blockstore.NewBlockstore(bds)
cacheOpts := blockstore.DefaultCacheOpts()
cacheOpts.HasBloomFilterSize = 0
@ -310,6 +320,21 @@ var importBenchCmd = &cli.Command{
pprof.StopCPUProfile()
if true {
resp, err := http.Get("http://localhost:6060/debug/metrics/prometheus")
if err != nil {
return err
}
metricsfi, err := os.Create("import-bench.metrics")
if err != nil {
return err
}
io.Copy(metricsfi, resp.Body) //nolint:errcheck
metricsfi.Close() //nolint:errcheck
}
return nil
},

4
go.mod
View File

@ -12,7 +12,7 @@ require (
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b
github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07
github.com/coreos/go-systemd/v22 v22.0.0
github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e
github.com/dgraph-io/badger/v2 v2.2007.2
@ -81,6 +81,7 @@ require (
github.com/ipfs/go-log v1.0.4
github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4
github.com/ipfs/go-merkledag v0.3.2
github.com/ipfs/go-metrics-prometheus v0.0.2
github.com/ipfs/go-path v0.0.7
github.com/ipfs/go-unixfs v0.2.4
github.com/ipfs/interface-go-ipfs-core v0.2.3
@ -115,6 +116,7 @@ require (
github.com/multiformats/go-multihash v0.0.14
github.com/opentracing/opentracing-go v1.2.0
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
github.com/prometheus/client_golang v1.6.0
github.com/raulk/clock v1.1.0
github.com/stretchr/testify v1.6.1
github.com/supranational/blst v0.1.1

4
go.sum
View File

@ -136,6 +136,8 @@ github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOi
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b h1:OKALTB609+19AM7wsO0k8yMwAqjEIppcnYvyIhA+ZlQ=
github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ=
github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07 h1:Cb2pZUCFXlLA8i7My+wrN51D41GeuhYOKa1dJeZt6NY=
github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ=
github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbLi0587YXpi8tOJT52qCOI/1I0UNThc/I=
github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
@ -630,6 +632,8 @@ github.com/ipfs/go-merkledag v0.3.2 h1:MRqj40QkrWkvPswXs4EfSslhZ4RVPRbxwX11js0t1
github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
github.com/ipfs/go-path v0.0.3/go.mod h1:zIRQUez3LuQIU25zFjC2hpBTHimWx7VK5bjZgRLbbdo=
github.com/ipfs/go-path v0.0.7 h1:H06hKMquQ0aYtHiHryOMLpQC1qC3QwXwkahcEVD51Ho=
github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno=