diff --git a/.circleci/config.yml b/.circleci/config.yml index c5bdeaa1b..0cff77eb2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,7 +68,7 @@ jobs: - run: sudo apt-get install npm - restore_cache: name: restore go mod cache - key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/lotus/go.mod" }} + key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }} - run: command: make buildall - store_artifacts: diff --git a/Makefile b/Makefile index 04f88a6b8..e37dcf3e8 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +SHELL=/usr/bin/env bash + all: build .PHONY: all @@ -77,6 +79,9 @@ lotus-shed: $(BUILD_DEPS) BINS+=lotus-seal-worker build: lotus lotus-storage-miner lotus-seal-worker + @[[ $$(type -P "lotus") ]] && echo "Caution: you have \ +an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true + .PHONY: build install: diff --git a/build/version.go b/build/version.go index 5b2079a61..10832a76a 100644 --- a/build/version.go +++ b/build/version.go @@ -22,7 +22,7 @@ const ( MajorOnlyMask = 0xff0000 MinorOnlyMask = 0x00ff00 PatchOnlyMask = 0x0000ff -c) +) // VersionInts returns (major, minor, patch) versions func VersionInts(version uint32) (uint32, uint32, uint32) { diff --git a/chain/store/store.go b/chain/store/store.go index 69816c5a2..83f957264 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -155,6 +155,8 @@ func (cs *ChainStore) SubHeadChanges(ctx context.Context) chan []*HeadChange { go func() { defer close(out) + var unsubOnce sync.Once + for { select { case val, ok := <-subch: @@ -170,7 +172,9 @@ func (cs *ChainStore) SubHeadChanges(ctx context.Context) chan []*HeadChange { case <-ctx.Done(): } case <-ctx.Done(): - go cs.bestTips.Unsub(subch) + unsubOnce.Do(func() { + go cs.bestTips.Unsub(subch) + }) } } }() diff --git a/cli/client.go b/cli/client.go index 0f1fdc991..61848951a 100644 --- a/cli/client.go +++ b/cli/client.go @@ -236,7 +236,7 @@ var clientRetrieveCmd = &cli.Command{ } if err := api.ClientRetrieve(ctx, offers[0].Order(payer), cctx.Args().Get(1)); err != nil { - return err + return xerrors.Errorf("Retrieval Failed: %w", err) } fmt.Println("Success") diff --git a/cli/params.go b/cli/params.go index df7e240ec..11a956120 100644 --- a/cli/params.go +++ b/cli/params.go @@ -1,6 +1,7 @@ package cli import ( + "github.com/docker/go-units" "github.com/filecoin-project/lotus/build" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -10,13 +11,18 @@ var fetchParamCmd = &cli.Command{ Name: "fetch-params", Usage: "Fetch proving parameters", Flags: []cli.Flag{ - &cli.Uint64Flag{ + &cli.StringFlag{ Name: "proving-params", - Usage: "download params used creating proofs for given size", + Usage: "download params used creating proofs for given size, i.e. 32GiB", }, }, Action: func(cctx *cli.Context) error { - err := build.GetParams(cctx.Uint64("proving-params")) + sectorSizeInt, err := units.FromHumanSize(cctx.String("proving-params")) + if err != nil { + return err + } + sectorSize := uint64(sectorSizeInt) + err = build.GetParams(sectorSize) if err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 7b9b48fad..54aa6c6b4 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "github.com/docker/go-units" "io/ioutil" "math/big" "math/rand" @@ -65,9 +66,10 @@ func main() { Value: "~/.lotus-bench", Usage: "Path to the storage directory that will store sectors long term", }, - &cli.Uint64Flag{ + &cli.StringFlag{ Name: "sector-size", - Value: 1024, + Value: "1GiB", + Usage: "size of the sectors in bytes, i.e. 32GiB", }, &cli.BoolFlag{ Name: "no-gpu", @@ -127,7 +129,11 @@ func main() { return err } - sectorSize := c.Uint64("sector-size") + sectorSizeInt, err := units.FromHumanSize(c.String("sector-size")) + if err != nil { + return err + } + sectorSize := uint64(sectorSizeInt) mds := datastore.NewMapDatastore() cfg := §orbuilder.Config{ diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index ce2cc64b9..c8d1b6b7e 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -47,6 +47,11 @@ var DaemonCmd = &cli.Command{ Name: "genesis", Usage: "genesis file to use for first node run", }, + &cli.StringFlag{ + Name: "genesis-timestamp", + Hidden: true, + Usage: "set the timestamp for the genesis block that will be created", + }, &cli.BoolFlag{ Name: "bootstrap", Value: true, @@ -84,7 +89,7 @@ var DaemonCmd = &cli.Command{ if cctx.String(preSealedSectorsFlag) == "" { return xerrors.Errorf("must also pass file with miner preseal info to `--%s`", preSealedSectorsFlag) } - genesis = node.Override(new(modules.Genesis), testing.MakeGenesis(cctx.String(makeGenFlag), cctx.String(preSealedSectorsFlag))) + genesis = node.Override(new(modules.Genesis), testing.MakeGenesis(cctx.String(makeGenFlag), cctx.String(preSealedSectorsFlag), cctx.String("genesis-timestamp"))) } var api api.FullNode diff --git a/documentation/cn/api-scripting-support.md b/documentation/cn/api-scripting-support.md new file mode 100644 index 000000000..4f16a070a --- /dev/null +++ b/documentation/cn/api-scripting-support.md @@ -0,0 +1,3 @@ +# API Scripting Support + +工作正在进行中 diff --git a/documentation/cn/updating-lotus.md b/documentation/cn/updating-lotus.md new file mode 100644 index 000000000..fcf1ec4f9 --- /dev/null +++ b/documentation/cn/updating-lotus.md @@ -0,0 +1,3 @@ +# Updating Lotus + +工作正在进行中 \ No newline at end of file diff --git a/documentation/en/api-scripting-support.md b/documentation/en/api-scripting-support.md new file mode 100644 index 000000000..27bc59aa5 --- /dev/null +++ b/documentation/en/api-scripting-support.md @@ -0,0 +1,31 @@ +# API Scripting Support + +You may want to delegate the work **Lotus Storage Miner** or **Lotus Node** perform to other machines. Here is how to setup the necessary authorization and environment variables. + +## Generate a JWT + +To generate a JWT for your environment variables, use this command: + +```sh +lotus auth create-token --perm admin +lotus-storage-miner auth create-token --perm admin +``` + +## Environment variables + +Environmental variables are variables that are defined for the current shell and are inherited by any child shells or processes. Environmental variables are used to pass information into processes that are spawned from the shell. + +Using the JWT you generated, you can assign it and the **multiaddr** to the appropriate environment variable. + +```sh +# Lotus Node +FULLNODE_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/1234/http" + +# Lotus Storage Miner +STORAGE_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/2345/http" +``` + +* The **Lotus Node**'s `mutliaddr` is in `~/.lotus/api`. +* The default token is in `~/.lotus/token`. +* The **Lotus Storage Miner**'s `multiaddr` is in `~/.lotusstorage/config`. +* The default token is in `~/.lotusstorage/token`. \ No newline at end of file diff --git a/documentation/en/api-troubleshooting.md b/documentation/en/api-troubleshooting.md index 9282764a3..72f3d81ae 100644 --- a/documentation/en/api-troubleshooting.md +++ b/documentation/en/api-troubleshooting.md @@ -17,7 +17,7 @@ curl -X POST \ ## Sending a CID -If you do not serialize the CID as a JSON IPLD link, you will receive an error. Here is an example of a broken CURL request: +If you do not serialize the CID as a [JSON IPLD link](https://did-ipid.github.io/ipid-did-method/#txref), you will receive an error. Here is an example of a broken CURL request: ```sh curl -X POST \ diff --git a/documentation/en/api.md b/documentation/en/api.md index bb6791206..cb1f9a1f7 100644 --- a/documentation/en/api.md +++ b/documentation/en/api.md @@ -6,7 +6,7 @@ Implementation details for the **JSON-RPC** package are [here](https://github.co ## Overview -API requests are made against `127.0.0.1:1234` unless you modify `~/.lotus/api`. +API requests are made against `127.0.0.1:1234` unless you modify `.lotus/config.toml`. Options: @@ -16,7 +16,11 @@ Options: ## What methods can I use? -Every `method` is available in [api/api.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). +For now, you can look into different files to find methods available to you based on your needs: + +* [Both Lotus node + storage miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go) +* [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) +* [Storage miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go) The necessary permissions for each are in [api/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/struct.go). @@ -49,7 +53,7 @@ curl -X POST \ > In the future we will add a playground to make it easier to build and experiment with API requests. -## Authorization +## CURL authorization To authorize your request, you will need to include the **JWT** in a HTTP header, for example: @@ -59,23 +63,23 @@ To authorize your request, you will need to include the **JWT** in a HTTP header Admin token is stored in `~/.lotus/token` for the **Lotus Node** or `~/.lotusstorage/token` for the **Lotus Storage Miner**. -## Authorization types +## How do I generate a token? + +To generate a JWT with custom permissions, use this command: + +```sh +# Lotus Node +lotus auth create-token --perm admin + +# Lotus Storage Miner +lotus-storage-miner auth create-token --perm admin +``` + +## What authorization level should I use? When viewing [api/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/struct.go), you will encounter these types: - `read` - Read node state, no private data. -- `write` - Write to local store / chain, read private data. -- `sign` - Use private keys stored in wallet for signing. -- `admin` - Manage permissions. - -Payload - -```json -{ - "Allow": [ - "read", - "write", - /* other options */ - ] -} -``` +- `write` - Write to local store / chain, and `read` permissions. +- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions. +- `admin` - Manage permissions, `read`, `write`, and `sign` permissions. diff --git a/documentation/en/hardware-mining.md b/documentation/en/hardware-mining.md index 4e047c50d..045a00607 100644 --- a/documentation/en/hardware-mining.md +++ b/documentation/en/hardware-mining.md @@ -1,5 +1,46 @@ # Mining Hardware -> This page is a work in progress. EVERYTHING HERE CAN CHANGE AND WILL CHANGE. PURCHASE HARDWARE AT YOUR OWN RISK. +> This documentation page describes the standard testing configuration the Protocol Labs team has used to test **Lotus Storage Miner**s on Lotus. There is no guarantee this testing configuration will be suitable for Filecoin storage mining at MainNet launch. If you need to buy new hardware to join the Filecoin TestNet, we recommend to buy no more hardware than you require for testing. To learn more please read this [hardware blog post](https://filecoin.io/blog/filecoin-testnet-mining/) -Please check out this [GitHub issue](https://github.com/filecoin-project/lotus/issues/694) to see benchmarks from existing hardware setups if you plan on participating in the **Filecoin TestNet** \ No newline at end of file +**Sector sizes** and **minimum pledged storage** required to mine blocks are two very important Filecoin TestNet parameters that impact hardware decisions. We will continue to refine all parameters during TestNet. + +BECAUSE OF THIS, OUR STANDARD TESTING CONFIGURATION FOR FILECOIN MAINNET CAN AND WILL CHANGE. YOU HAVE BEEN WARNED. + +## Example configuration + +The setup below is a minimal example for sealing 32 GiB sectors on Lotus: + +* 3 TB of hard drive space. +* 8 to 12 core CPU +* 256 GB of RAM + +## Benchmarked GPUs + +GPUs are a must for getting **block rewards**. Here are a few that have been tried in the past: + +* GeForce RTX 2080 Ti +* GeForce RTX 2080 SUPER +* GeForce RTX 2080 +* GeForce GTX 1080 Ti +* GeForce GTX 1080 +* GeForce GTX 1060 + +## Testing other GPUs + +If you want to test other GPUs, such as the GeForce GTX 1660, you can use the following configuration flag: + +```sh +BELLMAN_CUSTOM_GPU=":" +``` + +Here is an example of trying a GeForce GTX 1660 ti with 1536 cores. + +```sh +BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536" +``` + +To get the number of cores for your GPU, you will need to check your cards specifications. + +## Benchmarking + +Here is a [benchmarking tool](https://github.com/filecoin-project/lotus/tree/testnet-staging/cmd/lotus-bench) and a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694) for those who wish to experiment with and contribute hardware setups for the **Filecoin TestNet**. \ No newline at end of file diff --git a/documentation/en/join-devnet.md b/documentation/en/join-devnet.md index ac8c8dd1c..8efba8e5e 100644 --- a/documentation/en/join-devnet.md +++ b/documentation/en/join-devnet.md @@ -49,13 +49,13 @@ Here is an example of the response t3vhfme4qfvegqaz7m7q6o6afjcs67n6kpzv7t2eozio4chwpafwa2y4l7zhwd5eom7jmihzdg4s52dpvnclza ``` -- Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/funds.html) +- Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/funds.html) to add funds. - Paste the address you created. - Press the send button. ## Check wallet address balance -Wallet balances in the devnet are in **FIL**, the smallest denomination of FIL is an **attoFil**, where 1 attoFil = 10^-18 FIL. +Wallet balances in the Lotus DevNet are in **FIL**, the smallest denomination of FIL is an **attoFil**, where 1 attoFil = 10^-18 FIL. ```sh lotus wallet balance @@ -63,6 +63,14 @@ lotus wallet balance You will not see any attoFIL in your wallet if your **chain** is not fully synced. +## Send FIL to another wallet + +To send FIL to another wallet, use this command: + +``` +lotus send +``` + ## Monitor the dashboard To see the latest network activity, including **chain block height**, **block height**, **blocktime**, **total network power**, largest **block producer miner**, check out the [monitoring dashboard](https://lotus-metrics.kittyhawk.wtf). diff --git a/documentation/en/mining.md b/documentation/en/mining.md index e9b338c64..38c81176f 100644 --- a/documentation/en/mining.md +++ b/documentation/en/mining.md @@ -17,7 +17,7 @@ lotus wallet list With your wallet address: - Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/miner.html) -- Click "Create Miner +- Click "Create Miner" - DO NOT REFRESH THE PAGE. THIS OPERATION CAN TAKE SOME TIME. The task will be complete when you see: @@ -50,6 +50,8 @@ To mine: lotus-storage-miner run ``` +If you are downloading **Filecoin Proof Parameters**, the download can take some time. + Get information about your miner: ```sh diff --git a/documentation/en/retrieving-data.md b/documentation/en/retrieving-data.md index 7f86873e7..56f8296e8 100644 --- a/documentation/en/retrieving-data.md +++ b/documentation/en/retrieving-data.md @@ -2,7 +2,7 @@ > There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can. -Here are the operations you can perform after you have stored a **Data CID** with the **Lotus Storage Miner** in the network. +Here are the operations you can perform after you have stored and sealed a **Data CID** with the **Lotus Storage Miner** in the network. If you would like to learn how to store a **Data CID** on a miner, read the instructions [here](https://docs.lotu.sh/en+storing-data). diff --git a/documentation/en/setup-troubleshooting.md b/documentation/en/setup-troubleshooting.md index 63d7234f9..c14d0a996 100644 --- a/documentation/en/setup-troubleshooting.md +++ b/documentation/en/setup-troubleshooting.md @@ -23,6 +23,12 @@ ERROR hello hello/hello.go:81 other peer has different genesis! * Try deleting your file system's `~/.lotus` directory. Check that it exists with `ls ~/.lotus`. +```sh +- repo is already locked +``` + +* You already have another lotus deamon running. + ## Failed messages Some errors will occur that do not prevent Lotus from working: diff --git a/documentation/en/storing-data-troubleshooting.md b/documentation/en/storing-data-troubleshooting.md index d39e73cf7..19f77934f 100644 --- a/documentation/en/storing-data-troubleshooting.md +++ b/documentation/en/storing-data-troubleshooting.md @@ -10,4 +10,14 @@ WARN main lotus/main.go:72 routing: not found WARN main lotus/main.go:72 failed to start deal: computing commP failed: generating CommP: Piece must be at least 127 bytes ``` -* There is a minimum file size of 127 bytes. \ No newline at end of file +* There is a minimum file size of 127 bytes. + +## Troubleshooting Sealing + +Miners can check sealing progress with this command: + +```sh +lotus-storage-miner sectors list +``` + +When sealing is complete, `pSet: NO` will become `pSet: YES`. From now on the **Data CID** is [retrievable](https://docs.lotu.sh/en+retrieving-data) from the **Lotus Storage Miner**. \ No newline at end of file diff --git a/documentation/en/storing-data.md b/documentation/en/storing-data.md index 723088cd7..ef178f94d 100644 --- a/documentation/en/storing-data.md +++ b/documentation/en/storing-data.md @@ -52,4 +52,4 @@ lotus client deal Upon success, this command will return a **Deal CID**. -From now on the **Data CID** is [retrievable](https://docs.lotu.sh/en+retrieving-data) from the **Lotus Storage Miner**. \ No newline at end of file +To retrieve this file it must be sealed. If the **Lotus Storage Miner** is not running on a machine designed for sealing, the process will take a very long time. diff --git a/documentation/en/updating-lotus.md b/documentation/en/updating-lotus.md index bfb9ec065..0c721884c 100644 --- a/documentation/en/updating-lotus.md +++ b/documentation/en/updating-lotus.md @@ -3,7 +3,11 @@ If you installed Lotus on your machine, you can upgrade to the latest version by doing the following: ```sh +# get the latest git pull origin master + +# clean and remake the binaries +make clean build ``` Sometimes when you run Lotus after a pull, certain commands such as `lotus daemon` may break. diff --git a/go.mod b/go.mod index 1dfb070ee..942e2f284 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/GeertJohan/go.rice v1.0.0 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/docker/go-units v0.4.0 github.com/fatih/color v1.7.0 // indirect github.com/filecoin-project/chain-validation v0.0.3 github.com/filecoin-project/filecoin-ffi v0.0.0-20191204125133-ebb3e13addf1 diff --git a/go.sum b/go.sum index 9edc75a8c..45d7afb48 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhY github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 319143785..84b505fce 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -692,15 +692,15 @@ func fallbackPostChallengeCount(sectors uint64) uint64 { } func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error { - if err := migrate(osb.cacheDir, sb.cacheDir, true); err != nil { + if err := migrate(osb.cacheDir, sb.cacheDir, symlink); err != nil { return err } - if err := migrate(osb.sealedDir, sb.sealedDir, true); err != nil { + if err := migrate(osb.sealedDir, sb.sealedDir, symlink); err != nil { return err } - if err := migrate(osb.stagedDir, sb.stagedDir, true); err != nil { + if err := migrate(osb.stagedDir, sb.stagedDir, symlink); err != nil { return err } diff --git a/lib/statestore/store.go b/lib/statestore/store.go index 994938223..a334b60b8 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" cbg "github.com/whyrusleeping/cbor-gen" + "go.uber.org/multierr" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/lib/cborutil" @@ -139,6 +140,8 @@ func (st *StateStore) List(out interface{}) error { outT := reflect.TypeOf(out).Elem().Elem() rout := reflect.ValueOf(out) + var errs error + for { res, ok := res.NextSync() if !ok { @@ -151,7 +154,8 @@ func (st *StateStore) List(out interface{}) error { elem := reflect.New(outT) err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface()) if err != nil { - return err + errs = multierr.Append(errs, xerrors.Errorf("decoding state for key '%s': %w", res.Key, err)) + continue } rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem())) diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index f0195b639..5efe22382 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -68,7 +68,7 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo } } -func MakeGenesis(outFile, presealInfo string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { +func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") @@ -119,7 +119,18 @@ func MakeGenesis(outFile, presealInfo string) func(bs dtypes.ChainBlockstore, w addrs[miner.Worker] = types.FromFil(100000) } - b, err := gen.MakeGenesisBlock(bs, addrs, gmc, uint64(time.Now().Unix())) + ts := uint64(time.Now().Unix()) + if timestamp != "" { + t, err := time.Parse(time.RFC3339, timestamp) + if err != nil { + return nil, xerrors.Errorf("parsing input genesis timestamp: %w", err) + } + + glog.Infof("will use %s as the genesis timestamp", t) + ts = uint64(t.Unix()) + } + + b, err := gen.MakeGenesisBlock(bs, addrs, gmc, ts) if err != nil { return nil, err } diff --git a/retrieval/client.go b/retrieval/client.go index 405320a46..cfbfdfb05 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -100,7 +100,7 @@ type clientStream struct { func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, total types.BigInt, miner peer.ID, client, minerAddr address.Address, out io.Writer) error { s, err := c.h.NewStream(ctx, miner, ProtocolID) if err != nil { - return err + return xerrors.Errorf("failed to open stream to miner for retrieval query: %w", err) } defer s.Close() @@ -149,7 +149,6 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, cst.offset += toFetch } - log.Info("RETRIEVE SUCCESSFUL") return nil } @@ -173,13 +172,12 @@ func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out } if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil { - return err + return xerrors.Errorf("sending incremental retrieval request: %w", err) } var resp DealResponse if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil { - log.Error(err) - return err + return xerrors.Errorf("reading retrieval response: %w", err) } if resp.Status != Accepted { diff --git a/storage/sectorblocks/blockstore.go b/storage/sectorblocks/blockstore.go index de6039ff5..b0807ef3a 100644 --- a/storage/sectorblocks/blockstore.go +++ b/storage/sectorblocks/blockstore.go @@ -2,6 +2,8 @@ package sectorblocks import ( "context" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/storage" "io/ioutil" blocks "github.com/ipfs/go-block-format" @@ -72,11 +74,22 @@ func (s *SectorBlockStore) Get(c cid.Cid) (blocks.Block, error) { return nil, blockstore.ErrNotFound } - best := refs[0] // TODO: better strategy (e.g. look for already unsealed) - - si, err := s.sectorBlocks.Miner.GetSectorInfo(best.SectorID) - if err != nil { - return nil, xerrors.Errorf("getting sector info: %w", err) + // TODO: better strategy (e.g. look for already unsealed) + var best api.SealedRef + var bestSi storage.SectorInfo + for _, r := range refs { + si, err := s.sectorBlocks.Miner.GetSectorInfo(r.SectorID) + if err != nil { + return nil, xerrors.Errorf("getting sector info: %w", err) + } + if si.State == api.Proving { + best = r + bestSi = si + break + } + } + if bestSi.State == api.UndefinedSectorState { + return nil, xerrors.New("no sealed sector found") } log.Infof("reading block %s from sector %d(+%d;%d)", c, best.SectorID, best.Offset, best.Size) @@ -85,8 +98,8 @@ func (s *SectorBlockStore) Get(c cid.Cid) (blocks.Block, error) { best.SectorID, best.Offset, best.Size, - si.Ticket.TicketBytes, - si.CommD, + bestSi.Ticket.TicketBytes, + bestSi.CommD, ) if err != nil { return nil, xerrors.Errorf("unsealing block: %w", err) diff --git a/storage/sectors.go b/storage/sectors.go index 8dad777cc..d52dc70b6 100644 --- a/storage/sectors.go +++ b/storage/sectors.go @@ -65,7 +65,7 @@ func (m *Miner) UpdateSectorState(ctx context.Context, sector uint64, snonce uin func (m *Miner) sectorStateLoop(ctx context.Context) error { trackedSectors, err := m.ListSectors() if err != nil { - return xerrors.Errorf("loading sector list: %w", err) + log.Errorf("loading sector list: %+v", err) } go func() { diff --git a/storage/sectors_test.go b/storage/sectors_test.go new file mode 100644 index 000000000..858e3876e --- /dev/null +++ b/storage/sectors_test.go @@ -0,0 +1,56 @@ +package storage + +import ( + "bytes" + "testing" + + "gotest.tools/assert" + + "github.com/filecoin-project/lotus/lib/cborutil" +) + +func TestSectorInfoSelialization(t *testing.T) { + si := &SectorInfo{ + State: 123, + SectorID: 234, + Nonce: 345, + Pieces: []Piece{{ + DealID: 1234, + Size: 5, + CommP: []byte{3}, + }}, + CommD: []byte{32, 4}, + CommR: nil, + Proof: nil, + Ticket: SealTicket{ + BlockHeight: 345, + TicketBytes: []byte{87, 78, 7, 87}, + }, + PreCommitMessage: nil, + Seed: SealSeed{}, + CommitMessage: nil, + FaultReportMsg: nil, + LastErr: "hi", + } + + b, err := cborutil.Dump(si) + if err != nil { + t.Fatal(err) + } + + var si2 SectorInfo + if err := cborutil.ReadCborRPC(bytes.NewReader(b), &si); err != nil { + return + } + + assert.Equal(t, si.State, si2.State) + assert.Equal(t, si.Nonce, si2.Nonce) + assert.Equal(t, si.SectorID, si2.SectorID) + + assert.Equal(t, si.Pieces, si2.Pieces) + assert.Equal(t, si.CommD, si2.CommD) + assert.Equal(t, si.Ticket, si2.Ticket) + + assert.Equal(t, si, si2) + +} diff --git a/tools/stats/chain.dashboard.json b/tools/stats/chain.dashboard.json index 9effbf85f..be923cc33 100644 --- a/tools/stats/chain.dashboard.json +++ b/tools/stats/chain.dashboard.json @@ -14,7 +14,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "6.4.2" + "version": "6.5.0-pre" }, { "type": "panel", @@ -60,6 +60,457 @@ "id": null, "links": [], "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_INFLUXDB}", + "decimals": 2, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 0 + }, + "hideTimeOverride": false, + "id": 38, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_miner", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "miner" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.election", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT moving_average(count(\"value\"), 10) FROM \"chain.election\" WHERE $timeFilter -10m GROUP BY time($__interval), \"miner\" fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + }, + { + "params": [ + "20" + ], + "type": "moving_average" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blocks Won", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_INFLUXDB}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*/", + "color": "rgb(31, 120, 193)" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [], + "measurement": "chain.power", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Power", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": 0 + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [], + "measurement": "chain.blocktime", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT difference(mean(\"value\")) FROM \"chain.blocktime\" WHERE $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Avg Blocktime", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": 0 + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [], + "measurement": "chain.block_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Blocks In Tipset", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, { "cacheTimeout": null, "colorBackground": false, @@ -83,7 +534,7 @@ "h": 3, "w": 4, "x": 0, - "y": 0 + "y": 20 }, "id": 4, "interval": null, @@ -181,7 +632,7 @@ "h": 3, "w": 4, "x": 4, - "y": 0 + "y": 20 }, "id": 14, "interval": null, @@ -261,104 +712,6 @@ ], "valueName": "current" }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_INFLUXDB}", - "format": "decbytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 22, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true, - "ymax": null, - "ymin": 0 - }, - "tableColumn": "", - "targets": [ - { - "groupBy": [], - "measurement": "chain.power", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - } - ] - ], - "tags": [] - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Total Power", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, { "cacheTimeout": null, "colorBackground": false, @@ -380,8 +733,8 @@ "gridPos": { "h": 3, "w": 4, - "x": 12, - "y": 0 + "x": 8, + "y": 20 }, "id": 32, "interval": null, @@ -495,8 +848,8 @@ "gridPos": { "h": 3, "w": 4, - "x": 16, - "y": 0 + "x": 12, + "y": 20 }, "id": 20, "interval": null, @@ -610,8 +963,8 @@ "gridPos": { "h": 3, "w": 4, - "x": 20, - "y": 0 + "x": 16, + "y": 20 }, "id": 8, "interval": null, @@ -704,6 +1057,124 @@ ], "valueName": "avg" }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 20 + }, + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "pluginVersion": "6.4.2", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": 0 + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "measurement": "chain.message_count", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"value\") FROM (SELECT \"value\" FROM \"chain.message_count\" GROUP BY \"height\") WHERE $timeFilter GROUP BY time($__interval) fill(0)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Avg Messages in Tipset", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, { "cacheTimeout": null, "colorBackground": false, @@ -727,7 +1198,7 @@ "h": 3, "w": 4, "x": 0, - "y": 3 + "y": 23 }, "id": 16, "interval": "", @@ -810,73 +1281,71 @@ "valueName": "current" }, { + "aliasColors": {}, + "bars": false, "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], + "dashLength": 10, + "dashes": false, "datasource": "${DS_INFLUXDB}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 3, - "w": 4, + "w": 16, "x": 4, - "y": 3 + "y": 23 }, - "id": 6, - "interval": null, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, "links": [], - "mappingType": 1, - "mappingTypes": [ + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ { - "name": "value to text", - "value": 1 + "alias": "chain.height.difference", + "yaxis": 2 }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "Null Blocks", + "yaxis": 2 + }, { - "from": "null", - "text": "N/A", - "to": "null" + "alias": "Block Time", + "color": "rgb(31, 120, 193)" } ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true, - "ymax": null, - "ymin": 0 - }, - "tableColumn": "", + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { + "alias": "Block Time", "groupBy": [], - "measurement": "chain.block_count", + "measurement": "chain.blocktime", "orderByTime": "ASC", "policy": "default", + "query": "SELECT difference(\"value\") FROM \"chain.blocktime\" WHERE $timeFilter", + "rawQuery": true, "refId": "A", "resultFormat": "time_series", "select": [ @@ -886,26 +1355,87 @@ "value" ], "type": "field" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "tags": [] + }, + { + "alias": "Null Blocks", + "groupBy": [], + "measurement": "chain.height", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "difference" + }, + { + "params": [ + "-1" + ], + "type": "math" } ] ], "tags": [] } ], - "thresholds": "", + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Blocks In Tipset", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "title": "", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { - "op": "=", - "text": "N/A", - "value": "null" + "format": "s", + "label": "Time delta", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": "# null blocks", + "logBase": 1, + "max": null, + "min": "0", + "show": true } ], - "valueName": "current" + "yaxis": { + "align": false, + "alignLevel": null + } }, { "cacheTimeout": null, @@ -928,8 +1458,8 @@ "gridPos": { "h": 3, "w": 4, - "x": 8, - "y": 3 + "x": 20, + "y": 23 }, "id": 30, "interval": null, @@ -1005,381 +1535,6 @@ ], "valueName": "current" }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_INFLUXDB}", - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 3 - }, - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true, - "ymax": null, - "ymin": 0 - }, - "tableColumn": "", - "targets": [ - { - "groupBy": [], - "measurement": "chain.blocktime", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT difference(mean(\"value\")) FROM \"chain.blocktime\" WHERE $timeFilter GROUP BY time($__interval) fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "difference" - } - ] - ], - "tags": [] - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Avg Blocktime", - "type": "singlestat", - "valueFontSize": "100%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_INFLUXDB}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 3 - }, - "id": 10, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "pluginVersion": "6.4.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true, - "ymax": null, - "ymin": 0 - }, - "tableColumn": "", - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "chain.message_count", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"value\") FROM (SELECT \"value\" FROM \"chain.message_count\" GROUP BY \"height\") WHERE $timeFilter GROUP BY time($__interval) fill(0)", - "rawQuery": true, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Avg Messages in Tipset", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_INFLUXDB}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "chain.height.difference", - "yaxis": 2 - }, - { - "alias": "Null Blocks", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Block Time", - "groupBy": [], - "measurement": "chain.blocktime", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT difference(\"value\") FROM \"chain.blocktime\" WHERE $timeFilter", - "rawQuery": true, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "difference" - } - ] - ], - "tags": [] - }, - { - "alias": "Null Blocks", - "groupBy": [], - "measurement": "chain.height", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "difference" - }, - { - "params": [ - "-1" - ], - "type": "math" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Tipsets", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": "Time between tipsets", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": 0, - "format": "short", - "label": "Number of Null blocks", - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, { "columns": [], "datasource": "${DS_INFLUXDB}", @@ -1388,7 +1543,7 @@ "h": 16, "w": 4, "x": 0, - "y": 10 + "y": 26 }, "id": 28, "options": {}, @@ -1472,15 +1627,15 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_INFLUXDB}", - "fill": 5, + "fill": 4, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 4, - "y": 10 + "y": 26 }, - "id": 26, + "id": 40, "interval": "", "legend": { "alignAsTable": true, @@ -1613,7 +1768,7 @@ "h": 16, "w": 8, "x": 16, - "y": 10 + "y": 26 }, "id": 18, "options": {}, @@ -1710,7 +1865,7 @@ "h": 8, "w": 12, "x": 4, - "y": 18 + "y": 34 }, "id": 24, "legend": { @@ -1735,7 +1890,12 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*/", + "color": "rgb(31, 120, 193)" + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, @@ -1830,7 +1990,7 @@ "h": 9, "w": 12, "x": 0, - "y": 26 + "y": 42 }, "id": 34, "legend": { @@ -1856,7 +2016,7 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { @@ -1890,6 +2050,8 @@ "measurement": "chain.message_count", "orderByTime": "ASC", "policy": "default", + "query": "SELECT moving_average(count(\"value\"), 2) FROM \"chain.message_count\" WHERE $timeFilter -4m GROUP BY time($__interval), \"actor\", \"method\" fill(null)", + "rawQuery": true, "refId": "A", "resultFormat": "time_series", "select": [ @@ -1956,25 +2118,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_INFLUXDB}", + "decimals": 2, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 26 + "y": 42 }, "id": 36, + "interval": "", "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": false, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sort": "avg", + "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -1988,7 +2154,7 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { @@ -2000,6 +2166,12 @@ ], "type": "time" }, + { + "params": [ + "exitcode" + ], + "type": "tag" + }, { "params": [ "actor" @@ -2012,12 +2184,6 @@ ], "type": "tag" }, - { - "params": [ - "exitcode" - ], - "type": "tag" - }, { "params": [ "null" @@ -2028,6 +2194,8 @@ "measurement": "chain.message_count", "orderByTime": "ASC", "policy": "default", + "query": "SELECT moving_average(count(\"value\"), 10) FROM \"chain.message_count\" WHERE $timeFilter -10m GROUP BY time($__interval), \"exitcode\", \"actor\", \"method\" fill(null)", + "rawQuery": true, "refId": "A", "resultFormat": "time_series", "select": [ @@ -2041,6 +2209,12 @@ { "params": [], "type": "count" + }, + { + "params": [ + 10 + ], + "type": "moving_average" } ] ], @@ -2067,6 +2241,7 @@ }, "yaxes": [ { + "decimals": null, "format": "short", "label": "", "logBase": 1, @@ -2089,7 +2264,7 @@ } } ], - "refresh": "5s", + "refresh": "30s", "schemaVersion": 20, "style": "dark", "tags": [], @@ -2117,5 +2292,5 @@ "timezone": "", "title": "Chain", "uid": "z6FtI92Zz", - "version": 12 + "version": 8 } \ No newline at end of file diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 8d63f1c6d..652002ca1 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + "github.com/multiformats/go-multihash" _ "github.com/influxdata/influxdb1-client" models "github.com/influxdata/influxdb1-client/models" @@ -124,8 +125,11 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti if err != nil { return err } + p := NewPoint("chain.election", 1) + p.AddTag("miner", blockheader.Miner.String()) + pl.AddPoint(p) - p := NewPoint("chain.blockheader_size", len(bs)) + p = NewPoint("chain.blockheader_size", len(bs)) pl.AddPoint(p) } @@ -212,7 +216,13 @@ func RecordTipsetMessagesPoints(ctx context.Context, api api.FullNode, pl *Point } p = NewPoint("chain.message_count", 1) - p.AddTag("actor", actor.Code.String()) + + dm, err := multihash.Decode(actor.Code.Hash()) + if err != nil { + continue + } + + p.AddTag("actor", string(dm.Digest)) p.AddTag("method", fmt.Sprintf("%d", msg.Message.Method)) p.AddTag("exitcode", fmt.Sprintf("%d", recp[i].ExitCode)) pl.AddPoint(p)