From cac848c106fb1608944958d4105c9186f957e42e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 31 Aug 2020 14:24:23 -0700 Subject: [PATCH 01/80] add a command to import an ipld object into the chainstore --- cmd/lotus-shed/import-car.go | 57 ++++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + 2 files changed, 58 insertions(+) diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index 01343c4a3..9cbff953b 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -1,10 +1,13 @@ package main import ( + "encoding/hex" "fmt" "io" "os" + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" "github.com/ipld/go-car" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -81,3 +84,57 @@ var importCarCmd = &cli.Command{ } }, } + +var importObjectCmd = &cli.Command{ + Name: "import-obj", + Usage: "import a raw ipld object into your datastore", + Action: func(cctx *cli.Context) error { + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.FullNode) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + ds, err := lr.Datastore("/chain") + if err != nil { + return err + } + + bs := blockstore.NewBlockstore(ds) + + c, err := cid.Decode(cctx.Args().Get(0)) + if err != nil { + return err + } + + data, err := hex.DecodeString(cctx.Args().Get(1)) + if err != nil { + return err + } + + blk, err := block.NewBlockWithCid(data, c) + if err != nil { + return err + } + + if err := bs.Put(blk); err != nil { + return err + } + + return nil + + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 5438a31ef..11b98a3ac 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -24,6 +24,7 @@ func main() { bigIntParseCmd, staterootCmd, importCarCmd, + importObjectCmd, commpToCidCmd, fetchParamCmd, proofsCmd, From f58e8bc9a393c7fb3aca463fad961b45c8a0b8f4 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 1 Sep 2020 02:18:02 -0400 Subject: [PATCH 02/80] Fix some failed precommit handling --- extern/storage-sealing/states_failed.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extern/storage-sealing/states_failed.go b/extern/storage-sealing/states_failed.go index e313fd712..25f60ff2f 100644 --- a/extern/storage-sealing/states_failed.go +++ b/extern/storage-sealing/states_failed.go @@ -37,16 +37,16 @@ func (m *Sealing) checkPreCommitted(ctx statemachine.Context, sector SectorInfo) tok, _, err := m.api.ChainHead(ctx.Context()) if err != nil { log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err) - return nil, true + return nil, false } info, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) if err != nil { log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err) - return nil, true + return nil, false } - return info, false + return info, true } func (m *Sealing) handleSealPrecommit1Failed(ctx statemachine.Context, sector SectorInfo) error { @@ -107,7 +107,7 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI } if pci, is := m.checkPreCommitted(ctx, sector); is && pci != nil { - if sector.PreCommitMessage != nil { + if sector.PreCommitMessage == nil { log.Warn("sector %d is precommitted on chain, but we don't have precommit message", sector.SectorNumber) return ctx.Send(SectorPreCommitLanded{TipSet: tok}) } From fe52c475703ad417178667477c003e6cd5d930bc Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 31 Aug 2020 17:11:01 +0200 Subject: [PATCH 03/80] Docs review and re-organization This: * Re-organizes the docs into sections that align with what docs.filecoin.io becoming: * An installation section * A "getting started" section (lotus client focused) * A "storing" section (lotus client focused) * A "mining" section (miner focused) * A "build" section (developer focused) * An legacy "architecture" section is left in the last place. A few high-value documentation pages have been reviewed and updated with the latest recommendations: * Installation section and lotus setup * Miner setup * etc. ... Other pages have been correctly merged into the new relevant sections. Some pages have not been touched. The filesystem layout of the documentation has been changed into folders corresponding to the sections (as requested by @cw). Some pages that were not linked at all and/or where hidden, have been moved to "unclassified". This should make the porting of the Lotus documentation to docs.filecoin.io much easier, while ensuring it is more up to date than it was before. For the moment, this breaks most links as link-aliasing is not supported in lotus-docs. --- Makefile | 2 +- documentation/en/.library.json | 353 +++++++++--------- documentation/en/about.md | 14 + documentation/en/api-scripting-support.md | 25 -- documentation/en/api.md | 85 ----- .../en/{ => architecture}/architecture.md | 2 +- documentation/en/{ => architecture}/mpool.md | 0 .../en/{ => building}/api-methods.md | 0 .../en/{ => building}/api-troubleshooting.md | 0 documentation/en/building/api.md | 38 ++ documentation/en/building/building.md | 5 + .../jaeger-tracing.md} | 0 .../local-devnet.md} | 0 .../en/{ => building}/payment-channels.md | 0 documentation/en/building/remote-api.md | 69 ++++ documentation/en/cli.md | 108 ------ documentation/en/dev-tools.md | 3 - documentation/en/environment-vars.md | 65 ---- documentation/en/faqs.md | 36 +- documentation/en/getting-started.md | 23 -- .../en/getting-started/getting-started.md | 3 + .../setup-troubleshooting.md | 13 +- documentation/en/getting-started/setup.md | 169 +++++++++ documentation/en/getting-started/wallet.md | 58 +++ documentation/en/hardware-mining.md | 54 --- documentation/en/hardware.md | 7 - documentation/en/install-lotus-arch.md | 51 --- documentation/en/install-lotus-fedora.md | 54 --- documentation/en/install-lotus-ubuntu.md | 54 --- documentation/en/install-systemd-services.md | 145 ------- .../en/installation/install-linux.md | 129 +++++++ .../install-macos.md} | 2 +- documentation/en/installation/installation.md | 39 ++ documentation/en/installation/update.md | 72 ++++ documentation/en/join-testnet.md | 93 ----- documentation/en/miner-deals.md | 39 -- documentation/en/mining.md | 149 -------- documentation/en/mining/gpus.md | 17 + .../lotus-seal-worker.md} | 58 +-- documentation/en/mining/managing-deals.md | 19 + documentation/en/mining/miner-setup.md | 241 ++++++++++++ .../en/{ => mining}/mining-troubleshooting.md | 9 +- documentation/en/mining/mining.md | 8 + documentation/en/retrieving-data.md | 27 -- documentation/en/setting-a-static-port.md | 54 --- .../adding-from-ipfs.md} | 6 +- documentation/en/store/making-deals.md | 71 ++++ documentation/en/store/retrieve.md | 27 ++ .../storage-troubleshooting.md} | 13 +- documentation/en/store/store.md | 11 + documentation/en/storing-data.md | 62 --- .../WIP-arch-complementary-notes.md | 0 .../en/{ => unclassified}/block-validation.md | 0 .../en/{dev => unclassified}/create-miner.md | 0 .../{ => unclassified}/dev-tools-pond-ui.md | 0 .../en/{ => unclassified}/sealing-procs.md | 0 documentation/en/updating-lotus.md | 14 - 57 files changed, 1239 insertions(+), 1357 deletions(-) create mode 100644 documentation/en/about.md delete mode 100644 documentation/en/api-scripting-support.md delete mode 100644 documentation/en/api.md rename documentation/en/{ => architecture}/architecture.md (99%) rename documentation/en/{ => architecture}/mpool.md (100%) rename documentation/en/{ => building}/api-methods.md (100%) rename documentation/en/{ => building}/api-troubleshooting.md (100%) create mode 100644 documentation/en/building/api.md create mode 100644 documentation/en/building/building.md rename documentation/en/{dev-tools-jaeger-tracing.md => building/jaeger-tracing.md} (100%) rename documentation/en/{local-dev-net.md => building/local-devnet.md} (100%) rename documentation/en/{ => building}/payment-channels.md (100%) create mode 100644 documentation/en/building/remote-api.md delete mode 100644 documentation/en/cli.md delete mode 100644 documentation/en/dev-tools.md delete mode 100644 documentation/en/environment-vars.md delete mode 100644 documentation/en/getting-started.md create mode 100644 documentation/en/getting-started/getting-started.md rename documentation/en/{ => getting-started}/setup-troubleshooting.md (60%) create mode 100644 documentation/en/getting-started/setup.md create mode 100644 documentation/en/getting-started/wallet.md delete mode 100644 documentation/en/hardware-mining.md delete mode 100644 documentation/en/hardware.md delete mode 100644 documentation/en/install-lotus-arch.md delete mode 100644 documentation/en/install-lotus-fedora.md delete mode 100644 documentation/en/install-lotus-ubuntu.md delete mode 100644 documentation/en/install-systemd-services.md create mode 100644 documentation/en/installation/install-linux.md rename documentation/en/{install-lotus-macos.md => installation/install-macos.md} (85%) create mode 100644 documentation/en/installation/installation.md create mode 100644 documentation/en/installation/update.md delete mode 100644 documentation/en/join-testnet.md delete mode 100644 documentation/en/miner-deals.md delete mode 100644 documentation/en/mining.md create mode 100644 documentation/en/mining/gpus.md rename documentation/en/{mining-lotus-worker.md => mining/lotus-seal-worker.md} (60%) create mode 100644 documentation/en/mining/managing-deals.md create mode 100644 documentation/en/mining/miner-setup.md rename documentation/en/{ => mining}/mining-troubleshooting.md (90%) create mode 100644 documentation/en/mining/mining.md delete mode 100644 documentation/en/retrieving-data.md delete mode 100644 documentation/en/setting-a-static-port.md rename documentation/en/{storing-ipfs-integration.md => store/adding-from-ipfs.md} (79%) create mode 100644 documentation/en/store/making-deals.md create mode 100644 documentation/en/store/retrieve.md rename documentation/en/{storing-data-troubleshooting.md => store/storage-troubleshooting.md} (51%) create mode 100644 documentation/en/store/store.md delete mode 100644 documentation/en/storing-data.md rename documentation/en/{dev => unclassified}/WIP-arch-complementary-notes.md (100%) rename documentation/en/{ => unclassified}/block-validation.md (100%) rename documentation/en/{dev => unclassified}/create-miner.md (100%) rename documentation/en/{ => unclassified}/dev-tools-pond-ui.md (100%) rename documentation/en/{ => unclassified}/sealing-procs.md (100%) delete mode 100644 documentation/en/updating-lotus.md diff --git a/Makefile b/Makefile index 4f6ece417..2e91cfa65 100644 --- a/Makefile +++ b/Makefile @@ -280,7 +280,7 @@ method-gen: gen: type-gen method-gen docsgen: - go run ./api/docgen > documentation/en/api-methods.md + go run ./api/docgen > documentation/en/building/api-methods.md print-%: @echo $*=$($*) diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 3fab0df9b..87c7353c1 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -1,214 +1,207 @@ { "posts": [ { - "title": "Hardware Requirements", - "slug": "en+hardware", - "github": "en/hardware.md", + "title": "About Lotus", + "slug": "en+lotus", + "github": "en/about.md", + "value": null, + "posts": [] + }, + { + "title": "Installation", + "slug": "en+install", + "github": "en/installation/installation.md", "value": null, "posts": [ - { - "title": "Testing Configuration", - "slug": "en+hardware-mining", - "github": "en/hardware-mining.md", - "value": null - } + { + "title": "Linux installation", + "slug": "en+install-linux", + "github": "en/installation/install-linux.md", + "value": null + }, + { + "title": "MacOS installation", + "slug": "en+install-macos", + "github": "en/installation/install-macos.md", + "value": null + }, + { + "title": "Updating Lotus", + "slug": "en+update", + "github": "en/installation/update.md", + "value": null + } ] }, { - "title": "Setup", + "title": "Getting started", "slug": "en+getting-started", - "github": "en/getting-started.md", + "github": "en/getting-started/getting-started.md", "value": null, "posts": [ - { - "title": "Arch Linux Installation", - "slug": "en+install-lotus-arch", - "github": "en/install-lotus-arch.md", - "value": null - }, - { - "title": "Ubuntu Installation", - "slug": "en+install-lotus-ubuntu", - "github": "en/install-lotus-ubuntu.md", - "value": null - }, - { - "title": "Fedora Installation", - "slug": "en+install-lotus-fedora", - "github": "en/install-lotus-fedora.md", - "value": null - }, - { - "title": "MacOS Installation", - "slug": "en+install-lotus-macos", - "github": "en/install-lotus-macos.md", - "value": null - }, - { - "title": "Updating Lotus", - "slug": "en+updating-lotus", - "github": "en/updating-lotus.md", - "value": null - }, - { - "title": "Join Testnet", - "slug": "en+join-testnet", - "github": "en/join-testnet.md", - "value": null - }, - { - "title": "Use Lotus with systemd", - "slug": "en+install-systemd-services", - "github": "en/install-systemd-services.md", - "value": null - }, - { - "title": "Setup Troubleshooting", - "slug": "en+setup-troubleshooting", - "github": "en/setup-troubleshooting.md", - "value": null - }, - { - "title": "Environment Variables", - "slug": "en+env-vars", - "github": "en/environment-vars.md", - "value": null - } + { + "title": "Setting up Lotus", + "slug": "en+setup", + "github": "en/getting-started/setup.md", + "value": null + }, + { + + "title": "Obtaining and sending FIL", + "slug": "en+wallet", + "github": "en/getting-started/wallet.md", + "value": null + }, + { + "title": "Setup troubleshooting", + "slug": "en+setup-troubleshooting", + "github": "en/getting-started/setup-troubleshooting.md", + "value": null + } ] }, { - "title": "Architecture", - "slug": "en+arch", - "github": "en/architecture.md", + "title": "Storing and retrieving data", + "slug": "en+store", + "github": "en/store/store.md", "value": null, "posts": [ - { - "title": "The Message Pool", - "slug": "en+mpool", - "github": "en/mpool.md", - "value": null - } + { + "title": "Making storage deals", + "slug": "en+making-deals", + "github": "en/store/making-deals.md", + "value": null + }, + { + "title": "Adding data from IPFS", + "slug": "en+adding-from-ipfs", + "github": "en/store/adding-from-ipfs.md", + "value": null + }, + { + "title": "Retrieving data", + "slug": "en+retriving", + "github": "en/store/retrieve.md", + "value": null + }, + { + "title": "Storage Troubleshooting", + "slug": "en+storage-troubleshooting", + "github": "en/store/storage-troubleshooting.md", + "value": null + } ] }, { - "title": "Storage Mining", + "title": "Storage mining", "slug": "en+mining", - "github": "en/mining.md", + "github": "en/mining/mining.md", "value": null, "posts": [ - { - "title": "Lotus Worker", - "slug": "en+lotus-worker", - "github": "en/mining-lotus-worker.md", - "value": null - }, - { - "title": "Static Ports", - "slug": "en+setting-a-static-port", - "github": "en/setting-a-static-port.md", - "value": null - }, - { - "title": "Mining Troubleshooting", - "slug": "en+mining-troubleshooting", - "github": "en/mining-troubleshooting.md", - "value": null - } + { + "title": "Miner setup", + "slug": "en+miner-setup", + "github": "en/mining/miner-setup.md", + "value": null + }, + { + "title": "Managing deals", + "slug": "en+managing-deals", + "github": "en/mining/managing-deals.md", + "value": null + }, + { + "title": "Lotus Worker", + "slug": "en+lotus-worker", + "github": "en/mining/lotus-seal-worker.md", + "value": null + }, + { + "title": "Benchmarking GPUs", + "slug": "en+gpus", + "github": "en/mining/gpus.md", + "value": null + }, + { + "title": "Mining Troubleshooting", + "slug": "en+mining-troubleshooting", + "github": "en/mining/mining-troubleshooting.md", + "value": null + } ] }, { - "title": "Storing Data", - "slug": "en+storing-data", - "github": "en/storing-data.md", + "title": "Building", + "slug": "en+building", + "github": "en/building/building.md", "value": null, "posts": [ - { - "title": "Storage Troubleshooting", - "slug": "en+storing-data-troubleshooting", - "github": "en/storing-data-troubleshooting.md", - "value": null - }, - { - "title": "Information for Miners", - "slug": "en+info-for-miners", - "github": "en/miner-deals.md", - "value": null - }, - { - "title": "IPFS Integration", - "slug": "en+ipfs-client-integration", - "github": "en/storing-ipfs-integration.md", - "value": null - } + { + "title": "Setting up remote API access", + "slug": "en+remote-api", + "github": "en/building/remote-api.md", + "value": null, + "posts": [] + }, + { + "title": "API endpoints and methods", + "slug": "en+api", + "github": "en/building/api.md", + "value": null, + "posts": [] + }, + { + "title": "API Reference", + "slug": "en+api-methods", + "github": "en/building/api-methods.md", + "value": null, + "posts": [] + }, + + { + "title": "Payment Channels", + "slug": "en+payment-channels", + "github": "en/building/payment-channels.md", + "value": null, + "posts": [] + }, + + { + "title": "Running a local devnet", + "slug": "en+local-devnet", + "github": "en/building/local-devnet.md", + "value": null, + "posts": [] + }, + { + "title": "Jaeger Tracing", + "slug": "en+jaeger-tracing", + "github": "en/building/jaeger-tracing.md", + "value": null, + "posts": [] + }, + + { + "title": "API Troubleshooting", + "slug": "en+api-troubleshooting", + "github": "en/building/api-troubleshooting.md", + "value": null, + "posts": [] + } ] }, { - "title": "Retrieving Data", - "slug": "en+retrieving-data", - "github": "en/retrieving-data.md", - "value": null, - "posts": [] - }, - { - "title": "Payment Channels", - "slug": "en+payment-channels", - "github": "en/payment-channels.md", - "value": null, - "posts": [] - }, - { - "title": "Command Line Interface", - "slug": "en+cli", - "github": "en/cli.md", - "value": null, - "posts": [] - }, - { - "title": "API", - "slug": "en+api", - "github": "en/api.md", + "title": "Lotus Architecture (WIP)", + "slug": "en+arch", + "github": "en/architectiure/architecture.md", "value": null, "posts": [ - { - "title": "Remote API Support", - "slug": "en+api-scripting-support", - "github": "en/api-scripting-support.md", - "value": null - }, - { - "title": "API Methods", - "slug": "en+api-methods", - "github": "en/api-methods.md", - "value": null - }, - { - "title": "API Troubleshooting", - "slug": "en+api-troubleshooting", - "github": "en/api-troubleshooting.md", - "value": null - } - ] - }, - { - "title": "Developer Tools", - "slug": "en+dev-tools", - "github": "en/dev-tools.md", - "value": null, - "posts": [ - { - "title": "Setup Local Devnet", - "slug": "en+setup-local-dev-net", - "github": "en/local-dev-net.md", - "value": null, - "posts": [] - }, - { - "title": "Jaeger Tracing", - "slug": "en+dev-tools-jaeger-tracing", - "github": "en/dev-tools-jaeger-tracing.md", - "value": null, - "posts": [] - } + { + "title": "The Message Pool", + "slug": "en+mpool", + "github": "en/architecture/mpool.md", + "value": null + } ] }, { @@ -224,7 +217,7 @@ "github": "en/.glossary.json", "value": null, "custom": { - "glossary": true + "glossary": true }, "posts": [] } diff --git a/documentation/en/about.md b/documentation/en/about.md new file mode 100644 index 000000000..ee8536ac9 --- /dev/null +++ b/documentation/en/about.md @@ -0,0 +1,14 @@ +# Lotus + +Lotus is an implementation of the **Filecoin Distributed Storage Network**. + +The **Lotus Node** (and the mining applications) can be built to join any of the [Filecoin networks](https://docs.filecoin.io/how-to/networks/). + +For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://filecoin-project.github.io/specs/). + +## What can I learn here? + +* How to [install](en+installation) and [setup](en+setup) the Lotus software +* How to [store data on the Filecoin network](en+store) +* How to [setup a high performance FIL miner](en+miner-setup) +* How to [configure and access Lotus APIs](en+remote-api) diff --git a/documentation/en/api-scripting-support.md b/documentation/en/api-scripting-support.md deleted file mode 100644 index 653f144ed..000000000 --- a/documentation/en/api-scripting-support.md +++ /dev/null @@ -1,25 +0,0 @@ -# Remote API Support - -You may want to delegate the work **Lotus Miner** or **Lotus Node** performs to other machines. -Here is how to setup the necessary authorization and environment variables. - -## Environment variables - -Environmental variables are variables that are defined for the current shell and are inherited by any child shells or processes. Environmental variables are used to pass information into processes that are spawned from the shell. - -Using the [JWT you generated](https://lotu.sh/en+api#how-do-i-generate-a-token-18865), you can assign it and the **multiaddr** to the appropriate environment variable. - -```sh -# Lotus Node -FULLNODE_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/1234/http" - -# Lotus Miner -MINER_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/2345/http" -``` - -You can also use `lotus auth api-info --perm admin` to quickly create _API_INFO env vars - -- The **Lotus Node**'s `mutliaddr` is in `~/.lotus/api`. -- The default token is in `~/.lotus/token`. -- The **Lotus Miner**'s `multiaddr` is in `~/.lotusminer/config`. -- The default token is in `~/.lotusminer/token`. diff --git a/documentation/en/api.md b/documentation/en/api.md deleted file mode 100644 index 9760e2f32..000000000 --- a/documentation/en/api.md +++ /dev/null @@ -1,85 +0,0 @@ -# API - -Here is an early overview of how to make API calls. - -Implementation details for the **JSON-RPC** package are [here](https://github.com/filecoin-project/go-jsonrpc). - -## Overview: How do you modify the config.toml to change the API endpoint? - -API requests are made against `127.0.0.1:1234` unless you modify `.lotus/config.toml`. - -Options: - -- `http://[api:port]/rpc/v0` - HTTP endpoint -- `ws://[api:port]/rpc/v0` - Websocket endpoint -- `PUT http://[api:port]/rest/v0/import` - File import, it requires write permissions. - -## What methods can I use? - -For now, you can look into different files to find methods available to you based on your needs: - -- [Both Lotus node + miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go) -- [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) -- [Lotus miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go) - -The necessary permissions for each are in [api/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/struct.go). - -## How do I make an API request? - -To demonstrate making an API request, we will take the method `ChainHead` from [api/api_full.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). - -```go -ChainHead(context.Context) (*types.TipSet, error) -``` - -And create a CURL command. In this command, `ChainHead` is included as `{ "method": "Filecoin.ChainHead" }`: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -If the request requires authorization, add an authorization header: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $(cat ~/.lotusminer/token)" \ - --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -> In the future we will add a playground to make it easier to build and experiment with API requests. - -## CURL authorization - -To authorize your request, you will need to include the **JWT** in a HTTP header, for example: - -```sh --H "Authorization: Bearer $(cat ~/.lotusminer/token)" -``` - -Admin token is stored in `~/.lotus/token` for the **Lotus Node** or `~/.lotusminer/token` for the **Lotus Miner**. - -## How do I generate a token? - -To generate a JWT with custom permissions, use this command: - -```sh -# Lotus Node -lotus auth create-token --perm admin - -# Lotus Miner -lotus-miner auth create-token --perm admin -``` - -## What authorization level should I use? - -When viewing [api/apistruct/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/apistruct/struct.go), you will encounter these types: - -- `read` - Read node state, no private data. -- `write` - Write to local store / chain, and `read` permissions. -- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions. -- `admin` - Manage permissions, `read`, `write`, and `sign` permissions. diff --git a/documentation/en/architecture.md b/documentation/en/architecture/architecture.md similarity index 99% rename from documentation/en/architecture.md rename to documentation/en/architecture/architecture.md index 619e04f05..8c4d7be5c 100644 --- a/documentation/en/architecture.md +++ b/documentation/en/architecture/architecture.md @@ -6,7 +6,7 @@ Filecoin protocol, validating the blocks and state transitions. The specification for the Filecoin protocol can be found [here](https://filecoin-project.github.io/specs/). For information on how to setup and operate a Lotus node, -please follow the instructions [here](https://lotu.sh/en+getting-started). +please follow the instructions [here](en+getting-started). # Components diff --git a/documentation/en/mpool.md b/documentation/en/architecture/mpool.md similarity index 100% rename from documentation/en/mpool.md rename to documentation/en/architecture/mpool.md diff --git a/documentation/en/api-methods.md b/documentation/en/building/api-methods.md similarity index 100% rename from documentation/en/api-methods.md rename to documentation/en/building/api-methods.md diff --git a/documentation/en/api-troubleshooting.md b/documentation/en/building/api-troubleshooting.md similarity index 100% rename from documentation/en/api-troubleshooting.md rename to documentation/en/building/api-troubleshooting.md diff --git a/documentation/en/building/api.md b/documentation/en/building/api.md new file mode 100644 index 000000000..626193ee2 --- /dev/null +++ b/documentation/en/building/api.md @@ -0,0 +1,38 @@ +# API endpoints and methods + +The API can be accessed on: + +- `http://[api:port]/rpc/v0` - HTTP RPC-API endpoint +- `ws://[api:port]/rpc/v0` - Websocket RPC-API endpoint +- `PUT http://[api:port]/rest/v0/import` - REST endpoint for file import (multipart upload). It requires write permissions. + +The RPC methods can be found in the [Reference](en+api-methods) and directly in the source code: + +- [Both Lotus node + miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go) +- [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) +- [Lotus miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go) + + +## JSON-RPC client + +Lotus uses its own Go library implementation of [JSON-RPC](https://github.com/filecoin-project/go-jsonrpc). + +## cURL example + +To demonstrate making an API request, we will take the method `ChainHead` from [api/api.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). + +```go +ChainHead(context.Context) (*types.TipSet, error) +``` + +And create a CURL command. In this command, `ChainHead` is included as `{ "method": "Filecoin.ChainHead" }`: + +```sh +curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(cat ~/.lotusminer/token)" \ + --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \ + 'http://127.0.0.1:1234/rpc/v0' +``` + +(See [this section](en+remote-api) to learn how to generate authorization tokens). diff --git a/documentation/en/building/building.md b/documentation/en/building/building.md new file mode 100644 index 000000000..5194f8314 --- /dev/null +++ b/documentation/en/building/building.md @@ -0,0 +1,5 @@ +# Building with Lotus + +Lotus applications provide HTTP (JSON-RPC) APIs that allow developers to control Lotus programatically. + +This section dives into how to setup and use these APIs, additionally providing information on advanced Lotus features and workflows, like Payment Channels or how to setup a fully local Lotus development network. diff --git a/documentation/en/dev-tools-jaeger-tracing.md b/documentation/en/building/jaeger-tracing.md similarity index 100% rename from documentation/en/dev-tools-jaeger-tracing.md rename to documentation/en/building/jaeger-tracing.md diff --git a/documentation/en/local-dev-net.md b/documentation/en/building/local-devnet.md similarity index 100% rename from documentation/en/local-dev-net.md rename to documentation/en/building/local-devnet.md diff --git a/documentation/en/payment-channels.md b/documentation/en/building/payment-channels.md similarity index 100% rename from documentation/en/payment-channels.md rename to documentation/en/building/payment-channels.md diff --git a/documentation/en/building/remote-api.md b/documentation/en/building/remote-api.md new file mode 100644 index 000000000..d0fedb51b --- /dev/null +++ b/documentation/en/building/remote-api.md @@ -0,0 +1,69 @@ +# Setting up remote API access + +The **Lotus Miner** and the **Lotus Node** applications come with their own local API endpoints setup by default when they are running. + +These endpoints are used by `lotus` and `lotus-miner` to interact with the running process. In this section we will explain how to enable remote access to the Lotus APIs. + +Note that instructions are the same for `lotus` and `lotus-miner`. For simplicity, we will just show how to do it with `lotus`. + +## Setting the listening interface for the API endpoint + +By default, the API listens on the local "loopback" interface (`127.0.0.1`). This is configured in the `config.toml` file: + +```toml +[API] +# ListenAddress = "/ip4/127.0.0.1/tcp/1234/http" +# RemoteListenAddress = "" +# Timeout = "30s" +``` + +To access the API remotely, Lotus needs to listen on the right IP/interface. The IP associated to each interface can be usually found with the command `ip a`. Once the right IP is known, it can be set in the configuration: + +```toml +[API] +ListenAddress = "/ip4//tcp/3453/http" # port is an example + +# Only relevant for lotus-miner +# This should be the IP:Port pair where the miner is reachable from anyone trying to dial to it. +# If you have placed a reverse proxy or a NAT'ing device in front of it, this may be different from +# the EXTERNAL_INTERFACE_IP. +RemoteListenAddress = "" +``` + +> `0.0.0.0` can be used too. This is a wildcard that means "all interfaces". Depending on the network setup, this may affect security (listening on the wrong, exposed interface). + +After making these changes, please restart the affected process. + +## Issuing tokens + +Any client wishing to talk to the API endpoints will need a token. Tokens can be generated with: + +```sh +lotus auth create-token --perm +``` + +(similarly for the Lotus Miner). + +The permissions work as follows: + +- `read` - Read node state, no private data. +- `write` - Write to local store / chain, and `read` permissions. +- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions. +- `admin` - Manage permissions, `read`, `write`, and `sign` permissions. + + +Tokens can then be used in applications by setting an Authorization header as: + +``` +Authorization: Bearer +``` + + +## Environment variables + +`lotus`, `lotus-miner` and `lotus-worker` can actually interact with their respective applications running on a different node. All is needed to configure them are the following the *environment variables*: + +```sh +FULLNODE_API_INFO="TOKEN:/ip4//tcp//http" +MINER_API_INFO="TOKEN:/ip4//tcp//http" +``` diff --git a/documentation/en/cli.md b/documentation/en/cli.md deleted file mode 100644 index fd26400d0..000000000 --- a/documentation/en/cli.md +++ /dev/null @@ -1,108 +0,0 @@ -# Lotus Command Line Interface - -The Command Line Interface (CLI) is a convenient way to interact with -a Lotus node. You can use the CLI to operate your node, -get information about the blockchain, -manage your accounts and transfer funds, -create storage deals, and much more! - -The CLI is intended to be self-documenting, so when in doubt, simply add `--help` -to whatever command you're trying to run! This will also display all of the -input parameters that can be provided to a command. - -We highlight some of the commonly -used features of the CLI below. -All CLI commands should be run from the home directory of the Lotus project. - -## Operating a Lotus node - -### Starting up a node - -```sh -lotus daemon -``` -This command will start up your Lotus node, with its API port open at 1234. -You can pass `--api=` to use a different port. - -### Checking your sync progress - -```sh -lotus sync status -``` -This command will print your current tipset height under `Height`, and the target tipset height -under `Taregt`. - -You can also run `lotus sync wait` to get constant updates on your sync progress. - -### Getting the head tipset - -```sh -lotus chain head -``` - -### Control the logging level - -```sh -lotus log set-level -``` -This command can be used to toggle the logging levels of the different -systems of a Lotus node. In decreasing order -of logging detail, the levels are `debug`, `info`, `warn`, and `error`. - -As an example, -to set the `chain` and `blocksync` to log at the `debug` level, run -`lotus log set-level --system chain --system blocksync debug`. - -To see the various logging system, run `lotus log list`. - -### Find out what version of Lotus you're running - -```sh -lotus version -``` - -## Managing your accounts - -### Listing accounts in your wallet - -```sh -lotus wallet list -``` - -### Creating a new account - -```sh -lotus wallet new bls -``` -This command will create a new BLS account in your wallet; these -addresses start with the prefix `t3`. Running `lotus wallet new secp256k1` -(or just `lotus wallet new`) will create -a new Secp256k1 account, which begins with the prefix `t1`. - -### Getting an account's balance - -```sh -lotus wallet balance
-``` - -### Transferring funds - -```sh -lotus send --source= -``` -This command will transfer `amount` (in attoFIL) from `source address` to `destination address`. - -### Importing an account into your wallet - -```sh -lotus wallet import -``` -This command will import an account whose private key is saved at the specified file. - -### Exporting an account from your wallet - -```sh -lotus wallet export
-``` -This command will print out the private key of the specified address -if it is in your wallet. Always be careful with your private key! diff --git a/documentation/en/dev-tools.md b/documentation/en/dev-tools.md deleted file mode 100644 index 60b9b26d4..000000000 --- a/documentation/en/dev-tools.md +++ /dev/null @@ -1,3 +0,0 @@ -# Developer Tools - -> Running a local network can be a great way to understand how Lotus works and test your setup. diff --git a/documentation/en/environment-vars.md b/documentation/en/environment-vars.md deleted file mode 100644 index 9d455a74d..000000000 --- a/documentation/en/environment-vars.md +++ /dev/null @@ -1,65 +0,0 @@ -# Lotus Environment Variables - -## Building - -## Common - -The environment variables are common across most lotus binaries. - -### `LOTUS_FD_MAX` - -Sets the file descriptor limit for the process. This should be set high (8192 -or higher) if you ever notice 'too many open file descriptor' errors. - -### `LOTUS_JAEGER` - -This can be set to enable jaeger trace reporting. The value should be the url -of the jaeger trace collector, the default for most jaeger setups should be -`localhost:6831`. - -### `LOTUS_DEV` - -If set to a non-empty value, certain parts of the application will print more -verbose information to aid in development of the software. Not recommended for -end users. - -## Lotus Daemon - -### `LOTUS_PATH` - -Sets the location for the lotus daemon on-disk repo. If left empty, this defaults to `~/.lotus`. - -### `LOTUS_SKIP_GENESIS_CHECK` - -Can be set to `_yes_` if you wish to run a lotus network with a different -genesis than the default one built into your lotus binary. - -### `LOTUS_CHAIN_TIPSET_CACHE` - -Sets the cache size for the chainstore tipset cache. The default value is 8192, -but if your usage of the lotus API involves frequent arbitrary tipset lookups, -you may want to increase this. - -### `LOTUS_CHAIN_INDEX_CACHE` - -Sets the cache size for the chainstore epoch index cache. The default value is 32768, -but if your usage of the lotus API involves frequent deep chain lookups for -block heights that are very far from the current chain height, you may want to -increase this. - - -### `LOTUS_BSYNC_MSG_WINDOW` - -Set the initial maximum window size for message fetching blocksync requests. If -you have a slower internet connection and are having trouble syncing, you might -try lowering this down to 10-20 for a 'poor' internet connection. - -## Lotus Miner - -A number of environment variables are respected for configuring the behavior of the filecoin proving subsystem. For more details on those [see here](https://github.com/filecoin-project/rust-fil-proofs/#settings). - -### `LOTUS_MINER_PATH` - -Sets the location for the lotus miners on-disk repo. If left empty, this defaults to `~/.lotusminer`. - - diff --git a/documentation/en/faqs.md b/documentation/en/faqs.md index c2d526830..74119a5b6 100644 --- a/documentation/en/faqs.md +++ b/documentation/en/faqs.md @@ -11,7 +11,6 @@ go [here](https://filecoin.io/faqs/). Lotus is an implementation of the **Filecoin Distributed Storage Network**, written in Go. It is designed to be modular and interoperable with any other implementation of the Filecoin Protocol. -More information about Lotus can be found [here](https://lotu.sh/). ### What are the components of Lotus? @@ -30,21 +29,19 @@ to a Lotus Node over the JSON-RPC API. ### How do I set up a Lotus Node? -Follow the instructions found [here](https://lotu.sh/en+getting-started). +Follow the instructions found [here](en+install) and [here](en+setup). ### Where can I get the latest version of Lotus? -Download the binary tagged as the `Latest Release` from the - [Lotus Github repo](https://github.com/filecoin-project/lotus/releases). +Download the binary tagged as the `Latest Release` from the [Lotus Github repo](https://github.com/filecoin-project/lotus/releases) or checkout the `master` branch of the source repository. ### What operating systems can Lotus run on? -Lotus can build and run on most Linux and MacOS systems with at least -8GB of RAM. Windows is not yet supported. +Lotus can build and run on most Linux and MacOS systems with [at least 8GB of RAM](en+install#hardware-requirements-1). Windows is not yet supported. ### How can I update to the latest version of Lotus? -To update Lotus, follow the instructions [here](https://lotu.sh/en+updating-lotus). +To update Lotus, follow the instructions [here](en+update). ### How do I prepare a fresh installation of Lotus? @@ -52,7 +49,7 @@ Stop the Lotus daemon, and delete all related files, including sealed and chain running `rm ~/.lotus ~/.lotusminer`. Then, install Lotus afresh by following the instructions -found [here](https://lotu.sh/en+getting-started). +found [here](en+install). ### Can I configure where the node's config and data goes? @@ -73,48 +70,45 @@ directory for more. ### How can I send a request over the JSON-RPC API? Information on how to send a `cURL` request to the JSON-RPC API can be found -[here](https://lotu.sh/en+api). A JavaScript client is under development. +[here](en+api). ### What are the requests I can send over the JSON-RPC API? -Please have a look at the -[source code](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) -for a list of methods supported by the JSON-RPC API. +Please have a look [here](en+api). + + ## The Test Network ### What is Testnet? Testnet is a live network of Lotus Nodes run by the community for testing purposes. - It has 2 PiB of storage (and growing!) dedicated to it. ### Is FIL on the Testnet worth anything? -Nothing at all! Real-world incentives may be provided in a future phase of Testnet, but this is -yet to be confirmed. +Nothing at all! ### How can I see the status of Testnet? The [dashboard](https://stats.testnet.filecoin.io/) displays the status of the network as -well as a ton -of other metrics you might find interesting. +well as a ton of other metrics you might find interesting. ## Mining with a Lotus Node on Testnet ### How do I get started mining with Lotus? -Follow the instructions found [here](https://lotu.sh/en+mining). +Follow the instructions found [here](en+mining). ### What are the minimum hardware requirements? An example test configuration, and minimum hardware requirements can be found -[here](https://lotu.sh/en+hardware-mining). +[here](en+install#hardware-requirements-8). Note that these might NOT be the minimum requirements for mining on Mainnet. ### What are some GPUs that have been tested? -A list of benchmarked GPUs can be found [here](https://lotu.sh/en+hardware-mining#benchmarked-gpus-7393). +See previous question. ### Why is my GPU not being used when sealing a sector? @@ -135,4 +129,4 @@ You can do so by changing the storage path variable for the second miner, e.g., ### How do I setup my own local devnet? -Follow the instructions found [here](https://lotu.sh/en+setup-local-dev-net). +Follow the instructions found [here](en+local-devnet). diff --git a/documentation/en/getting-started.md b/documentation/en/getting-started.md deleted file mode 100644 index e38a2ab97..000000000 --- a/documentation/en/getting-started.md +++ /dev/null @@ -1,23 +0,0 @@ -# Lotus - -Lotus is an implementation of the **Filecoin Distributed Storage Network**. You can run the Lotus software client to join the **Filecoin Testnet**. - -For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://filecoin-project.github.io/specs/). - -## What can I learn here? - -- How to install Lotus on [Arch Linux](https://lotu.sh/en+install-lotus-arch), [Ubuntu](https://lotu.sh/en+install-lotus-ubuntu), or [MacOS](https://lotu.sh/en+install-lotus-macos). -- Joining the [Lotus Testnet](https://lotu.sh/en+join-testnet). -- [Storing](https://lotu.sh/en+storing-data) or [retrieving](https://lotu.sh/en+retrieving-data) data. -- Mining Filecoin using the **Lotus Miner** in your [CLI](https://lotu.sh/en+mining). - -## How is Lotus designed? - -Lotus is architected modularly to keep clean API boundaries while using the same process. Installing Lotus will include two separate programs: - -- The **Lotus Node** -- The **Lotus Miner** - -The **Lotus Miner** is intended to be run on the machine that manages a single miner instance, and is meant to communicate with the **Lotus Node** via the websocket **JSON-RPC** API for all of the chain interaction needs. - -This way, a mining operation may easily run a **Lotus Miner** or many of them, connected to one or many **Lotus Node** instances. diff --git a/documentation/en/getting-started/getting-started.md b/documentation/en/getting-started/getting-started.md new file mode 100644 index 000000000..99b4095d4 --- /dev/null +++ b/documentation/en/getting-started/getting-started.md @@ -0,0 +1,3 @@ +# Getting started + +This section will get you started with Lotus. We will setup the Lotus daemon (that should already be [installed](en+install)), start it, create a wallet and use it to send and receive some Filecoin. diff --git a/documentation/en/setup-troubleshooting.md b/documentation/en/getting-started/setup-troubleshooting.md similarity index 60% rename from documentation/en/setup-troubleshooting.md rename to documentation/en/getting-started/setup-troubleshooting.md index a1c78b51b..f27a3faa5 100644 --- a/documentation/en/setup-troubleshooting.md +++ b/documentation/en/getting-started/setup-troubleshooting.md @@ -1,5 +1,12 @@ # Setup Troubleshooting + +## Error: initializing node error: cbor input had wrong number of fields + +This happens when you are starting Lotus which has been compiled for one network, but it encounters data in the Lotus data folder which is for a different network, or for an older incompatible version. + +The solution is to clear the data folder (see below). + ## Config: Clearing data Here is a command that will delete your chain data, stored wallets, stored data and any miners you have set up: @@ -8,7 +15,7 @@ Here is a command that will delete your chain data, stored wallets, stored data rm -rf ~/.lotus ~/.lotusminer ``` -This command usually resolves any issues with running `lotus` but it is not always required for updates. We will share information about when resetting your chain data and miners is required for an update in the future. +Note you do not always need to clear your data for [updating](en+update). ## Error: Failed to connect bootstrap peer @@ -33,6 +40,8 @@ ERROR hello hello/hello.go:81 other peer has different genesis! ## Config: Open files limit +Lotus will attempt to set up the file descriptor (FD) limit automatically. If that does not work, you can still configure your system to allow higher than the default values. + On most systems you can check the open files limit with: ```sh @@ -44,3 +53,5 @@ You can also modify this number by using the `ulimit` command. It gives you the ```sh ulimit -n 10000 ``` + +Note that this is not persisted and that systemd manages its own FD limits for services. Please use your favourite search engine to find instructions on how to persist and configure FD limits for your system. diff --git a/documentation/en/getting-started/setup.md b/documentation/en/getting-started/setup.md new file mode 100644 index 000000000..e751da80b --- /dev/null +++ b/documentation/en/getting-started/setup.md @@ -0,0 +1,169 @@ +# Setting up Lotus + +Your Lotus binaries have been installed and you are ready to start participating in the Filecoin network. + +## Selecting the right network + +You should have built the Lotus binaries from the right Github branch and Lotus will be fully setup to join the matching [Filecoin network](https://docs.filecoin.io/how-to/networks/). For more information on switching networks, check the [updating Lotus section](en+update). + +## Starting the daemon + +To start the daemon simply run: + +```sh +lotus daemon +``` + +or if you are using the provided systemd service files, do: + +```sh +systemctl start lotus-daemon +``` + +__If you are using Lotus from China__, make sure you set the following environment variable before running Lotus: + +``` +export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" +``` + + +During the first start, Lotus: + +* Will setup its data folder at `~/.lotus` +* Will download the necessary parameters +* Start syncing the Lotus chain + +If you started lotus using systemd, the logs will appear in `/var/log/lotus/daemon.log` (not in journalctl as usual), otherwise you will see them in your screen. + +Do not be appalled by the amount of warnings and sometimes errors showing in the logs, there are usually part of the usual functioning of the daemon as part of a distributed network. + +## Waiting to sync + +After the first start, the chain will start syncing until it has reached the tip. You can check how far the syncing process is with: + +```sh +lotus sync status +``` + +You can also interactively wait for the chain to be fully synced with: + +```sh +lotus sync wait +``` + +## Interacting with the Lotus daemon + +As shown above, the `lotus` command allows to interact with the running daemon. You will see it getting used in many of the documentation examples. + +This command-line-interface is self-documenting: + +```sh +# Show general help +lotus --help +# Show specific help for the "client" subcommand +lotus client --help +``` + +For example, after your Lotus daemon has been running for a few minutes, use `lotus` to check the number of other peers that it is connected to in the Filecoin network: + +```sh +lotus net peers +``` + +## Controlling the logging level + +```sh +lotus log set-level +``` +This command can be used to toggle the logging levels of the different +systems of a Lotus node. In decreasing order +of logging detail, the levels are `debug`, `info`, `warn`, and `error`. + +As an example, +to set the `chain` and `blocksync` to log at the `debug` level, run +`lotus log set-level --system chain --system blocksync debug`. + +To see the various logging system, run `lotus log list`. + + +## Configuration + +### Configuration file + +The Lotus daemon stores a configuration file in `~/.lotus/config.toml`. Note that by default all settings are commented. Here is an example configuration: + +```toml +[API] + # Binding address for the Lotus API + ListenAddress = "/ip4/127.0.0.1/tcp/1234/http" + # Not used by lotus daemon + RemoteListenAddress = "" + # General network timeout value + Timeout = "30s" + +# Libp2p provides connectivity to other Filecoin network nodes +[Libp2p] + # Binding address swarm - 0 means random port. + ListenAddresses = ["/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0"] + # Insert any addresses you want to explicitally + # announce to other peers here. Otherwise, they are + # guessed. + AnnounceAddresses = [] + # Insert any addresses to avoid announcing here. + NoAnnounceAddresses = [] + # Connection manager settings, decrease if your + # machine is overwhelmed by connections. + ConnMgrLow = 150 + ConnMgrHigh = 180 + ConnMgrGrace = "20s" + +# Pubsub is used to broadcast information in the network +[Pubsub] + Bootstrapper = false + RemoteTracer = "/dns4/pubsub-tracer.filecoin.io/tcp/4001/p2p/QmTd6UvR47vUidRNZ1ZKXHrAFhqTJAD27rKL9XYghEKgKX" + +# This section can be used to enable adding and retriving files from IPFS +[Client] + UseIpfs = false + IpfsMAddr = "" + IpfsUseForRetrieval = false + +# Metrics configuration +[Metrics] + Nickname = "" + HeadNotifs = false +``` + +### Ensuring connectivity to your Lotus daemon + +Usually your lotus daemon will establish connectivity with others in the network and try to make itself diallable using uPnP. If you wish to manually ensure that your daemon is reachable: + +* Set a fixed port of your choice in the `ListenAddresses` in the Libp2p section (i.e. 6665). +* Open a port in your router that is forwarded to this port. This is usually called featured as "Port forwarding" and the instructions differ from router model to model but there are many guides online. +* Add your public IP/port to `AnnounceAddresses`. i.e. `/ip4//tcp/6665/`. + +Note that it is not a requirement to use Lotus as a client to the network to be fully reachable, as your node already connects to others directly. + + +### Environment variables + +Common to most Lotus binaries: + +* `LOTUS_FD_MAX`: Sets the file descriptor limit for the process +* `LOTUS_JAEGER`: Sets the Jaeger URL to send traces. See TODO. +* `LOTUS_DEV`: Any non-empty value will enable more verbose logging, useful only for developers. + +Specific to the *Lotus daemon*: + +* `LOTUS_PATH`: Location to store Lotus data (defaults to `~/.lotus`). +* `LOTUS_SKIP_GENESIS_CHECK=_yes_`: Set only if you wish to run a lotus network with a different genesis block. +* `LOTUS_CHAIN_TIPSET_CACHE`: Sets the size for the chainstore tipset cache. Defaults to `8192`. Increase if you perform frequent arbitrary tipset lookups. +* `LOTUS_CHAIN_INDEX_CACHE`: Sets the size for the epoch index cache. Defaults to `32768`. Increase if you perform frequent deep chain lookups for block heights far from the latest height. +* `LOTUS_BSYNC_MSG_WINDOW`: Set the initial maximum window size for message fetching blocksync request. Set to 10-20 if you have an internet connection with low bandwidth. + +Specific to the *Lotus miner*: + +* `LOTUS_MINER_PATH`: Location for the miner's on-disk repo. Defaults to `./lotusminer`. +* A number of environment variables are respected for configuring the behaviour of the Filecoin proving subsystem. [See here](en+miner-setup). + + diff --git a/documentation/en/getting-started/wallet.md b/documentation/en/getting-started/wallet.md new file mode 100644 index 000000000..25a67fb09 --- /dev/null +++ b/documentation/en/getting-started/wallet.md @@ -0,0 +1,58 @@ +# Obtaining and sending FIL + +In order to receive and send FIL with Lotus you will need to have installed the program and be running the Lotus daemon. + +## Creating a wallet + + +```sh +lotus wallet new bls +``` + +This will print your Filecoin address. + +Your wallet information is stored in the `~/.lotus/keystore` (or `$LOTUS_PATH/keystore`). For instructions on export/import, see below. + +You can create multiple wallets and list them with: + +```sh +lotus wallet list +``` + +## Obtaining FIL + +FIL can be obtained either by using one of the Faucets (available for the test networks) or by buying it from an exchange supporting FIL trading (once mainnet has launched). + +Once you have received some FIL you can check your balance with: + +```sh +lotus wallet balance +``` + +Remember that your will only see the latest balance when your daemon is fully synced to the chain. + +## Sending FIL + +Sending some FIL can be achieved by running: + +```sh +lotus wallet send
+``` + +Make sure to check `lotus wallet send --help` for additional options. + +## Exporting and importing a wallet + +You can export and re-import a wallet with: + +```sh +lotus wallet export
> wallet.private +``` + +and: + +```sh +lotus wallet import wallet.private +``` + +Keep your wallet's private key safe! diff --git a/documentation/en/hardware-mining.md b/documentation/en/hardware-mining.md deleted file mode 100644 index d421f6fb1..000000000 --- a/documentation/en/hardware-mining.md +++ /dev/null @@ -1,54 +0,0 @@ -# Protocol Labs Standard Testing Configuration - -> This documentation page describes the standard testing configuration the Protocol Labs team has used to test **Lotus Miner**s on Lotus. There is no guarantee this testing configuration will be suitable for Filecoin storage mining at MainNet launch. If you need to buy new hardware to join the Filecoin Testnet, we recommend to buy no more hardware than you require for testing. To learn more please read this [Protocol Labs Standard Testing Configuration post](https://filecoin.io/blog/filecoin-testnet-mining/). - -**Sector sizes** and **minimum pledged storage** required to mine blocks are two very important Filecoin Testnet parameters that impact hardware decisions. We will continue to refine all parameters during Testnet. - -BECAUSE OF THIS, OUR STANDARD TESTING CONFIGURATION FOR FILECOIN MAINNET CAN AND WILL CHANGE. YOU HAVE BEEN WARNED. - -## Example configuration - -The setup below is a minimal example for sealing 32 GiB sectors on Lotus: - -- 2 TB of hard drive space. -- 8 core CPU -- 128 GiB of RAM - -Note that 1GB sectors don't require as high of specs, but are likely to be removed as we improve the performance of 32GB sector sealing. - -For the first part of the sealing process, AMD CPU's are __highly recommended__, because of the `Intel SHA Extensions` instruction set that is available there ever since the `Zen` microarchitecture. Hence, AMD CPU's seem to perform much better on the testnet than other CPU's. Contrary to what the name implies, this extended instruction set is not available on recent Intel desktop/server chips. - -## Testnet discoveries - -- If you only have 128GiB of ram, enabling 256GB of **NVMe** swap on an SSD will help you avoid out-of-memory issues while mining. - -## Benchmarked GPUs - -GPUs are a must for getting **block rewards**. Here are a few that have been confirmed to generate **SNARKs** quickly enough to successfully mine blocks on the Lotus Testnet. - -- GeForce RTX 2080 Ti -- GeForce RTX 2080 SUPER -- GeForce RTX 2080 -- GeForce GTX 1080 Ti -- GeForce GTX 1080 -- GeForce GTX 1060 - -## Testing other GPUs - -If you want to test a GPU that is not explicitly supported, use the following global **environment variable**: - -```sh -BELLMAN_CUSTOM_GPU=":" -``` - -Here is an example of trying a GeForce GTX 1660 Ti with 1536 cores. - -```sh -BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536" -``` - -To get the number of cores for your GPU, you will need to check your card’s specifications. - -## Benchmarking - -Here is a [benchmarking tool](https://github.com/filecoin-project/lotus/tree/master/cmd/lotus-bench) and a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694) for those who wish to experiment with and contribute hardware setups for the **Filecoin Testnet**. diff --git a/documentation/en/hardware.md b/documentation/en/hardware.md deleted file mode 100644 index f6250548a..000000000 --- a/documentation/en/hardware.md +++ /dev/null @@ -1,7 +0,0 @@ -# Hardware - -> This page is a work in progress. Exact mining requirements are still in the works. - -Lotus can build and run on most [Linux](https://ubuntu.com/) and [MacOS](https://www.apple.com/macos) systems with at least 8GiB of RAM. - -Windows is not yet supported. diff --git a/documentation/en/install-lotus-arch.md b/documentation/en/install-lotus-arch.md deleted file mode 100644 index 8e06aae4e..000000000 --- a/documentation/en/install-lotus-arch.md +++ /dev/null @@ -1,51 +0,0 @@ -# Arch Linux Instructions - -These steps will install the following dependencies: - -- go (1.14 or higher) -- gcc (7.4.0 or higher) -- git (version 2 or higher) -- bzr (some go dependency needs this) -- jq -- pkg-config -- opencl-icd-loader -- opencl driver (like nvidia-opencl on arch) (for GPU acceleration) -- opencl-headers (build) -- rustup (proofs build) -- llvm (proofs build) -- clang (proofs build) - -### Install dependencies - -```sh -sudo pacman -Syu opencl-icd-loader gcc git bzr jq pkg-config opencl-icd-loader opencl-headers -``` - -### Install Go 1.14 - -Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install). - -### Clone the Lotus repository - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -### Build the Lotus binaries from source and install - -! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively** - -```sh -make clean && make all -sudo make install -``` - -#### Native Filecoin FFI building - -```sh -env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all -sudo make install -``` - -After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet). diff --git a/documentation/en/install-lotus-fedora.md b/documentation/en/install-lotus-fedora.md deleted file mode 100644 index c37161b7a..000000000 --- a/documentation/en/install-lotus-fedora.md +++ /dev/null @@ -1,54 +0,0 @@ -# Fedora Instructions - -> tested on 30 - -**NOTE:** If you have an AMD GPU the opencl instructions may be incorrect... - -These steps will install the following dependencies: - -- go (1.14 or higher) -- gcc (7.4.0 or higher) -- git (version 2 or higher) -- bzr (some go dependency needs this) -- jq -- pkg-config -- rustup (proofs build) -- llvm (proofs build) -- clang (proofs build) - -### Install dependencies - -```sh -$ sudo dnf -y update -$ sudo dnf -y install gcc git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm -$ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -``` - -### Install Go 1.14 - -Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install). - -### Clone the Lotus repository - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -### Build the Lotus binaries from source and install - -! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively** - -```sh -$ make clean && make all -$ sudo make install -``` - -#### Native Filecoin FFI building - -```sh -env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all -sudo make install -``` - -After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus TestNet](https://lotu.sh/en+join-testnet). diff --git a/documentation/en/install-lotus-ubuntu.md b/documentation/en/install-lotus-ubuntu.md deleted file mode 100644 index 500650692..000000000 --- a/documentation/en/install-lotus-ubuntu.md +++ /dev/null @@ -1,54 +0,0 @@ -# Ubuntu Instructions - -These steps will install the following dependencies: - -- go (1.14 or higher) -- gcc (7.4.0 or higher) -- git (version 2 or higher) -- bzr (some go dependency needs this) -- jq -- pkg-config -- opencl-icd-loader -- opencl driver (like nvidia-opencl on arch) (for GPU acceleration) -- opencl-headers (build) -- rustup (proofs build) -- llvm (proofs build) -- clang (proofs build) - -### Install dependencies - -```sh -sudo apt update -sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl -sudo apt upgrade -``` - -### Install Go 1.14 - -Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install). - -### Clone the Lotus repository - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -### Build the Lotus binaries from source and install - -! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively** - -```sh -make clean && make all -sudo make install -``` - -#### Native Filecoin FFI building - -```sh -env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all -sudo make install -``` - - -After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet). diff --git a/documentation/en/install-systemd-services.md b/documentation/en/install-systemd-services.md deleted file mode 100644 index fbde1feec..000000000 --- a/documentation/en/install-systemd-services.md +++ /dev/null @@ -1,145 +0,0 @@ -# Use Lotus with systemd - -Lotus is capable of running as a systemd service daemon. You can find installable service files for systemd in the [lotus repo scripts directory](https://github.com/filecoin-project/lotus/tree/master/scripts) as files with `.service` extension. In order to install these service files, you can copy these `.service` files to the default systemd unit load path. - -The services expect their binaries to be present in `/usr/local/bin/`. You can use `make` to install them by running: - -```sh -$ sudo make install -``` - -for `lotus` and `lotus-storage-miner` and - -```sh -$ sudo make install-chainwatch -``` - -for the `chainwatch` tool. - -## Installing services via `make` - -If your host uses the default systemd unit load path, the `lotus-daemon` and `lotus-miner` services can be installed by running: - -```sh -$ sudo make install-services -``` - -To install the the `lotus-chainwatch` service run: - -```sh -$ sudo make install-chainwatch-service -``` - -You can install all services together by running: - -```sh -$ sudo make install-all-services -``` - -The `lotus-daemon` and the `lotus-miner` services can be installed individually too by running: - -```sh -$ sudo make install-daemon-service -``` - -and - -```sh -$ sudo make install-miner-service -``` - -### Notes - -When installing the `lotus-miner` and/or `lotus-chainwatch` service the `lotus-daemon` service gets automatically installed since the other two services depend on it being installed to run. - -All `install-*-service*` commands will install the latest binaries in the lotus build folders to `/usr/local/bin/`. If you do not want to use the latest build binaries please copy the `*.service` files by hand. - -## Removing via `make` - -All services can beremoved via `make`. To remove all services together run: - -```sh -$ sudo make clean-all-services -``` - -Individual services can be removed by running: - -```sh -$ sudo make clean-chainwatch-services -$ sudo make clean-miner-services -$ sudo make clean-daemon-services -``` - -### Notes - -The services will be stoppend and disabled when removed. - -Removing the `lotus-daemon` service will automatically remove the depending services `lotus-miner` and `lotus-chainwatch`. - - -## Controlling services - -All service can be controlled with the `systemctl`. A few basic control commands are listed below. To get detailed infos about the capabilities of the `systemctl` command please consult your distributions man pages by running: - -```sh -$ man systemctl -``` - -### Start/Stop services - -You can start the services by running: - -```sh -$ sudo systemctl start lotus-daemon -$ sudo systemctl start lotus-miner -$ sudo systemctl start lotus-chainwatch -``` - -and can be stopped by running: - -```sh -$ sudo systemctl stop lotus-daemon -$ sudo systemctl stop lotus-miner -$ sudo systemctl stop lotus-chainwatch -``` - -### Enabling services on startup - -To enable the services to run automatically on startup execute: - -```sh -$ sudo systemctl enable lotus-daemon -$ sudo systemctl enable lotus-miner -$ sudo systemctl enable lotus-chainwatch -``` - -To disable the services on startup run: - -```sh -$ sudo systemctl disable lotus-daemon -$ sudo systemctl disable lotus-miner -$ sudo systemctl disable lotus-chainwatch -``` -### Notes - -Systemd will not let services be enabled or started without their requirements. Starting the `lotus-chainwatch` and/or `lotus-miner` service with automatically start the `lotus-daemon` service (if installed!). Stopping the `lotus-daemon` service will stop the other two services. The same pattern is executed for enabling and disabling the services. - -## Interacting with service logs - -Logs from the services can be reviewed using `journalctl`. - -### Follow logs from a specific service unit - -```sh -$ sudo journalctl -u lotus-daemon -f -``` - -### View logs in reverse order - -```sh -$ sudo journalctl -u lotus-miner -r -``` - -### Log files - -Besides the systemd service logs all services save their own log files in `/var/log/lotus/`. diff --git a/documentation/en/installation/install-linux.md b/documentation/en/installation/install-linux.md new file mode 100644 index 000000000..6fe12996e --- /dev/null +++ b/documentation/en/installation/install-linux.md @@ -0,0 +1,129 @@ +# Linux installation + +This page will show you the steps to build and install Lotus in your Linux computer. + +## Dependencies + +### System dependencies + +First of all, building Lotus will require installing some system dependencies, usually provided by your distribution. + +For Arch Linux: + +```sh +sudo pacman -Syu opencl-icd-loader gcc git bzr jq pkg-config opencl-icd-loader opencl-headers +``` + +For Ubuntu: + +```sh +sudo apt update +sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl +sudo apt upgrade +``` + +For Fedora: + +```sh +sudo dnf -y update +sudo dnf -y install gcc git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm +``` + +For OpenSUSE: + +```sh +sudo zypper in gcc git jq make libOpenCL1 opencl-headers ocl-icd-devel clang llvm +sudo ln -s /usr/lib64/libOpenCL.so.1 /usr/lib64/libOpenCL.so +``` + +### Rustup + +Lotus needs [rustup](https://rustup.rs/): + +```sh +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` + +Please make sure your `$PATH` variable is correctly configured after the rustup installation so that `cargo` and `rustc` are found in their rustup-configured locations. + +### Go + +To build lotus you will need a working installation of **[Go1.14](https://golang.org/dl/)**. Follow the [installation instructions](https://golang.org/doc/install), which generally amount to: + +```sh +# Example! Check the installation instructions. +wget -c https://dl.google.com/go/go1.14.7.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local +``` + +## Build and install Lotus + +With all the above, you are ready to build and install the Lotus suite (`lotus`, `lotus-miner` and `lotus-worker`): + +```sh +git clone https://github.com/filecoin-project/lotus.git +cd lotus/ +``` + +__IF YOU ARE IN CHINA__, set `export GOPROXY=https://goproxy.cn` before building + +Now, choose the network that you will be joining: + +* For `testnet`: `git checkout master` +* For `nerpa`: `git checkout ntwk-nerpa` +* For `butterfly`: `git checkout ntwk-butterfly` + +Once on the right branch, do: + +```sh +make clean install +sudo make install +``` + +This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`. `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets...). `lotus-miner` will use `$HOME/.lotusminer` respectively. See the *environment variables* section below for how to customize these. + +> Remeber to [move your Lotus folder](en+update) if you are switching between different networks, or there has been a network reset. + + +### Native Filecoin FFI + +Some newer processors (AMD Zen (and later), Intel Ice Lake) have support SHA extensions. To make full use of your processor's capabilities, make sure you set the following variables BEFORE building from source (as described above): + +```sh +export RUSTFLAGS="-C target-cpu=native -g" +export FFI_BUILD_FROM_SOURCE=1 +``` + +> __NOTE__: This method of building does not produce portable binaries! Make sure you run the binary in the same machine as you built it. + +### systemd service files + +Lotus provides Systemd service files. They can be installed with: + +```sh +make install-daemon-service +make install-miner-service +``` + +After that, you should be able to control Lotus using `systemctl`. + +## Troubleshooting + +This section mentions some of the common pitfalls for building Lotus. Check the [getting started](en+getting-started) section for more tips on issues when running the lotus daemon. + +### Build errors + +Please check the build logs closely. If you have a dirty state in your git branch make sure to: + +```sh +git checkout +git reset origin/ --hard +make clean +``` + +### Slow builds from China + +Users from China can speed up their builds by setting: + +```sh +export GOPROXY=https://goproxy.cn +``` diff --git a/documentation/en/install-lotus-macos.md b/documentation/en/installation/install-macos.md similarity index 85% rename from documentation/en/install-lotus-macos.md rename to documentation/en/installation/install-macos.md index 371832c96..ea9ecb8ca 100644 --- a/documentation/en/install-lotus-macos.md +++ b/documentation/en/installation/install-macos.md @@ -59,4 +59,4 @@ make clean && make all sudo make install ``` -After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet). +After intalling Lotus you will be ready to [setup and run the daemon](en+setup.md). diff --git a/documentation/en/installation/installation.md b/documentation/en/installation/installation.md new file mode 100644 index 000000000..98534da92 --- /dev/null +++ b/documentation/en/installation/installation.md @@ -0,0 +1,39 @@ +# Installation + +Lotus can be installed in [Linux](en-install-linux) and [MacOS](en-install-macos) machines by building it from source. Windows is not supported yet. + +This section contains guides to install Lotus in the supported platforms. + +Lotus is made of 3 binaries: + +* `lotus`: the main [Lotus node](en+setup) (Lotus client) +* `lotus-miner`: an application specifically for [Filecoin mining](en+miner-setup) +* `lotus-worker`: an additional [application to offload some heavy-processing tasks](en+lotus-worker) from the Lotus Miner. + +These applications are written in Go, but also import several Rust libraries. Lotus does not distribute +pre-compiled builds. + +## Hardware requirements + +### For client nodes + +* 8GiB of RAM +* Recommended for syncing speed: CPU with support for *Intel SHA Extensions* (AMD since Zen microarchitecture, Intel since Ice Lake). +* Recommended for speed: SSD hard drive (the bigger the better) + +### For miners + +The following correspond to the latest testing configuration: + +* 2 TB of hard drive space +* 8 core CPU +* 128 GiB of RAM with 256 GiB of NVMe SSD storage for swap (or simply, more RAM). +* Recommended for speed: CPU with support for *Intel SHA Extensions* (AMD since Zen microarchitecture, Intel since Ice Lake). +* GPU for block mining. The following have been [confirmed to be fast enough](en+gpus): + +- GeForce RTX 2080 Ti +- GeForce RTX 2080 SUPER +- GeForce RTX 2080 +- GeForce GTX 1080 Ti +- GeForce GTX 1080 +- GeForce GTX 1060 diff --git a/documentation/en/installation/update.md b/documentation/en/installation/update.md new file mode 100644 index 000000000..5d76592c9 --- /dev/null +++ b/documentation/en/installation/update.md @@ -0,0 +1,72 @@ +# Updating and restarting Lotus + +Updating Lotus is as simple as rebuilding and re-installing the software as explained in the previous sections. + +You can verify which version of Lotus you are running with: + +```sh +lotus version +``` + +Make sure that you `git pull` the branch that corresponds to the network that your Lotus daemon is using: + +```sh +git pull origin +make clean +make all +sudo make install # if necessary +``` + +Finally, restart the Lotus Node and/or Lotus Miner(s). + +__CAVEAT__: If you are running miners: check if your miner is safe to shut down and restart: `lotus-miner proving info`. If any deadline shows a block height in the past, do not restart: + +In the following example, Deadline Open is 454 which is earlier than Current Epoch of 500. This miner should **not** be shut down or restarted. + +``` +$ sudo lotus-miner proving info +Miner: t01001 +Current Epoch: 500 +Proving Period Boundary: 154 +Proving Period Start: 154 (2h53m0s ago) +Next Period Start: 3034 (in 21h7m0s) +Faults: 768 (100.00%) +Recovering: 768 +Deadline Index: 5 +Deadline Sectors: 0 +Deadline Open: 454 (23m0s ago) +Deadline Close: 514 (in 7m0s) +Deadline Challenge: 434 (33m0s ago) +Deadline FaultCutoff: 384 (58m0s ago) +``` + +In this next example, the miner can be safely restarted because no Deadlines are earlier than Current Epoch of 497. You have ~45 minutes before the miner must be back online to declare faults (FaultCutoff). If the miner has no faults, you have about an hour. + +``` +$ sudo lotus-miner proving info +Miner: t01000 +Current Epoch: 497 +Proving Period Boundary: 658 +Proving Period Start: 658 (in 1h20m30s) +Next Period Start: 3538 (in 25h20m30s) +Faults: 0 (0.00%) +Recovering: 0 +Deadline Index: 0 +Deadline Sectors: 768 +Deadline Open: 658 (in 1h20m30s) +Deadline Close: 718 (in 1h50m30s) +Deadline Challenge: 638 (in 1h10m30s) +Deadline FaultCutoff: 588 (in 45m30s) +``` + +## Switching networks and network resets + +If you wish to switch to a different lotus network or there has been a network reset, you will need to: + +* Checkout the appropiate repository branch and rebuild +* Ensure you do not mix Lotus data (`LOTUS_PATH`, usually `~/.lotus`) from a previous or different network. For this, either: + * Rename the folder to something else or, + * Set a different `LOTUS_PATH` for the new network. +* Same for `~/.lotusminer` if you are running a miner. + +Note that deleting the Lotus data folder will wipe all the chain data, wallets and configuration, so think twice before taking any non-reversible action. diff --git a/documentation/en/join-testnet.md b/documentation/en/join-testnet.md deleted file mode 100644 index 6660d26d8..000000000 --- a/documentation/en/join-testnet.md +++ /dev/null @@ -1,93 +0,0 @@ -# Join Testnet - -## Introduction - -Anyone can set up a **Lotus Node** and connect to the **Lotus Testnet**. This is the best way to explore the current CLI and the **Filecoin Decentralized Storage Market**. - -## Note: Using the Lotus Node from China - -If you are trying to use `lotus` from China. You should set this **environment variable** on your machine: - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - -## Get started - -Start the **daemon** using the default configuration in `./build`: - -```sh -lotus daemon -``` - -In another terminal window, check your connection with peers: - -```sh -lotus net peers | wc -l -``` - -In order to connect to the network, you need to be connected to at least 1 peer. If you’re seeing 0 peers, read our [troubleshooting notes](https://lotu.sh/en+setup-troubleshooting). - -Make sure that you have a reasonable "open files limit" set on your machine, such as 10000. If you're seeing a lower value, such as 256 (default on macOS), read our [troubleshooting notes](https://lotu.sh/en+setup-troubleshooting) on how to update it prior to starting the Lotus daemon. - -## Chain sync - -While the daemon is running, the next requirement is to sync the chain. Run the command below to view the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/). - -```sh -lotus sync wait -``` - -- This step will take anywhere between a few hours to a couple of days. -- You will be able to perform **Lotus Testnet** operations after it is finished. - -## Create your first address - -Initialize a new wallet: - -```sh -lotus wallet new -``` - -Sometimes your operating system may limit file name length to under 150 characters. You need to use a file system that supports long filenames. - -Here is an example of the response: - -```sh -t1aswwvjsae63tcrniz6x5ykvsuotlgkvlulnqpsi -``` - -- Visit the [faucet](http://spacerace.faucet.glif.io/) to add funds. -- Paste the address you created under REQUEST. -- Press the Request button. - -## Check wallet address balance - -Wallet balances in the Lotus Testnet are in **FIL**, the smallest denomination of FIL is an **attoFil**, where 1 attoFil = 10^-18 FIL. - -```sh -lotus wallet balance -``` - -You will not see any attoFIL in your wallet if your **chain** is not fully synced. - -## Send FIL to another wallet - -To send FIL to another wallet from your default account, use this command: - -``` -lotus send -``` - -## Configure your node's connectivity - -To effectively accept incoming storage & retrieval deals, your Lotus node needs to be accessible to other nodes on the network. To improve your connectivity, be sure to: - -- [Set the multiaddresses for you miner to listen on](https://docs.filecoin.io/mine/connectivity/#setting-multiaddresses) -- [Maintain a healthy peer count](https://docs.filecoin.io/mine/connectivity/#checking-peer-count) -- [Enable port forwarding](https://docs.filecoin.io/mine/connectivity/#port-forwarding) -- [Configure your public IP address and port](https://docs.filecoin.io/mine/connectivity/#setting-a-public-ip-address) - -## Monitor the dashboard - -To see the latest network activity, including **chain block height**, **block height**, **blocktime**, **total network power**, largest **block producer miner**, check out the [monitoring dashboard](https://stats.testnet.filecoin.io). diff --git a/documentation/en/miner-deals.md b/documentation/en/miner-deals.md deleted file mode 100644 index 0aee0e1af..000000000 --- a/documentation/en/miner-deals.md +++ /dev/null @@ -1,39 +0,0 @@ -# Information for Miners - -Here is how a miner can get set up to accept storage deals. The first step is -to install a Lotus node and sync to the top of the chain. - -## Set up an ask - -``` -lotus-miner set-price -``` - -This command will set up your miner to accept deal proposals that meet the input price. -The price is inputted in FIL per GiB per epoch, and the default is 0.0000000005. - -## Ensure you can be discovered - -Clients need to be able to find you in order to make storage deals with you. -While there isn't necessarily anything you need to do to become discoverable, here are some things you can -try to check that people can connect to you. - -To start off, make sure you are connected to at least some peers, and your port is -open and working. - -### Connect to your own node - -If you are in contact with someone else running Lotus, you can ask them to try connecting -to your node. To do so, provide them your peer ID, which you can get by running `lotus net id` on -your node. - -They can then try running `lotus net findpeer ` to get your address(es), and can then -run `lotus net connect
` to connect to you. If successful, your node will now -appear on their peers list (run `lotus net peers` to check). - -You can also check this by running a second instance of Lotus yourself. - -### Query your own ask - -A client should be able to find your ask by running `lotus client query-ask `. If -someone is not able to retrieve your ask by doing so, then there is an issue with your node. \ No newline at end of file diff --git a/documentation/en/mining.md b/documentation/en/mining.md deleted file mode 100644 index 32c3c51d2..000000000 --- a/documentation/en/mining.md +++ /dev/null @@ -1,149 +0,0 @@ -# Storage Mining - -Here are instructions to learn how to perform storage mining. For hardware specifications please read [this](https://lotu.sh/en+hardware-mining). - -It is useful to [join the Testnet](https://lotu.sh/en+join-testnet) prior to attempting storage mining for the first time. - -## Note: Using the Lotus Miner from China - -If you are trying to use `lotus-miner` from China. You should set this **environment variable** on your machine. - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - -## Get started - -Please ensure that at least one **BLS address** (starts with `t3`) in your wallet exists with the following command: - -```sh -lotus wallet list -``` - -If you do not have a bls address, create a new bls wallet: - -```sh -lotus wallet new bls -``` - -With your wallet address: - -- Visit the [faucet](http://spacerace.faucet.glif.io/) -- Paste the address you created under REQUEST. -- Press the Request button. -- Run `/lotus-miner init --owner= --worker=` - -You will have to wait some time for this operation to complete. - -## Mining - -To mine: - -```sh -lotus-miner run -``` - -If you are downloading **Filecoin Proof Parameters**, the download can take some time. - -Get information about your miner: - -```sh -lotus-miner info -# example: miner id `t0111` -``` - -**Seal** random data to start producing **PoSts**: - -```sh -lotus-miner sectors pledge -``` - -- Warning: On Linux configurations, this command will write data to `$TMPDIR` which is not usually the largest partition. You should point the value to a larger partition if possible. - -Get **miner power** and **sector usage**: - -```sh -lotus state power -# returns total power - -lotus state power - -lotus state sectors -``` - -## Performance tuning - -### `FIL_PROOFS_MAXIMIZE_CACHING=1` Environment variable - -This env var can be used with `lotus-miner`, `lotus-worker`, and `lotus-bench` to make the precommit1 step faster at the cost of some memory use (1x sector size) - -### `FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1` Environment variable - -This env var can be used with `lotus-miner`, `lotus-worker`, and `lotus-bench` to enable experimental precommit2 GPU acceleration - -### Setting multiaddresses - -Set multiaddresses for the miner to listen on in a miner's `config.toml` file -(by default, it is located at `~/.lotusminer/config.toml`). The `ListenAddresses` in this file should be interface listen addresses (usually `/ip4/0.0.0.0/tcp/PORT`), and the `AnnounceAddresses` should match the addresses being passed to `set-addrs`. - -The addresses passed to `set-addrs` parameter in the commands below should be currently active and dialable; confirm they are before entering them. - -Once the config file has been updated, set the on-chain record of the miner's listen addresses: - -``` -lotus-miner actor set-addrs ... -``` - -This updates the `MinerInfo` object in the miner's actor, which will be looked up -when a client attempts to make a deal. Any number of addresses can be provided. - -Example: - -``` -lotus-miner actor set-addrs /ip4/123.123.73.123/tcp/12345 /ip4/223.223.83.223/tcp/23456 -``` - -# Separate address for windowPoSt messages - -WindowPoSt is the mechanism through which storage is verified in Filecoin. It requires miners to submit proofs for all sectors every 24h, which require sending messages to the chain. - -Because many other mining related actions require sending messages to the chain, and not all of those are "high value", it may be desirable to use a separate account to send PoSt messages from. This allows for setting lower GasFeeCaps on the lower value messages without creating head-of-line blocking problems for the PoSt messages in congested chain conditions - -To set this up, first create a new account, and send it some funds for gas fees: -```sh -lotus wallet new bls -t3defg... - -lotus send t3defg... 100 -``` - -Next add the control address -```sh -lotus-miner actor control set t3defg... -Add t3defg... -Pass --really-do-it to actually execute this action -``` - -Now actually set the addresses -```sh -lotus-miner actor control set --really-do-it t3defg... -Add t3defg... -Message CID: bafy2.. -``` - -Wait for the message to land on chain -```sh -lotus state wait-msg bafy2.. -... -Exit Code: 0 -... -``` - -Check miner control address list to make sure the address was correctly setup -```sh -lotus-miner actor control list -name ID key use balance -owner t01111 t3abcd... other 300 FIL -worker t01111 t3abcd... other 300 FIL -control-0 t02222 t3defg... post 100 FIL -``` diff --git a/documentation/en/mining/gpus.md b/documentation/en/mining/gpus.md new file mode 100644 index 000000000..ad0ed4f66 --- /dev/null +++ b/documentation/en/mining/gpus.md @@ -0,0 +1,17 @@ +# Benchmarking additional GPUs + +If you want to test a GPU that is not explicitly supported, set the following *environment variable*: + +```sh +BELLMAN_CUSTOM_GPU=":" +``` + +Here is an example of trying a GeForce GTX 1660 Ti with 1536 cores. + +```sh +BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536" +``` + +To get the number of cores for your GPU, you will need to check your card’s specifications. + +To perform the benchmark you can use Lotus' [benchmarking tool](https://github.com/filecoin-project/lotus/tree/master/cmd/lotus-bench). Results and discussion are tracked in a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694). diff --git a/documentation/en/mining-lotus-worker.md b/documentation/en/mining/lotus-seal-worker.md similarity index 60% rename from documentation/en/mining-lotus-worker.md rename to documentation/en/mining/lotus-seal-worker.md index f93780c44..62dde9bfb 100644 --- a/documentation/en/mining-lotus-worker.md +++ b/documentation/en/mining/lotus-seal-worker.md @@ -2,24 +2,14 @@ The **Lotus Worker** is an extra process that can offload heavy processing tasks from your **Lotus Miner**. The sealing process automatically runs in the **Lotus Miner** process, but you can use the Worker on another machine communicating over a fast network to free up resources on the machine running the mining process. -## Note: Using the Lotus Worker from China +## Installation -If you are trying to use `lotus-worker` from China. You should set this **environment variable** on your machine: - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - -## Get Started - -Make sure that the `lotus-worker` is compiled and installed by running: - -```sh -make lotus-worker -``` +The `lotus-worker` application is installed along with the others when running `sudo make install` as shown in the [Installation section](en+install-linux). For simplicity, we recommend following the same procedure in the machines that will run the Lotus Workers (even if the Lotus miner and the Lotus daemon are not used there). ## Setting up the Miner +### Allow external connections to the miner API + First, you will need to ensure your `lotus-miner`'s API is accessible over the network. To do this, open up `~/.lotusminer/config.toml` (Or if you manually set `LOTUS_MINER_PATH`, look under that directory) and look for the API field. @@ -32,30 +22,49 @@ ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" RemoteListenAddress = "127.0.0.1:2345" ``` -To make your node accessible over the local area network, you will need to determine your machines IP on the LAN, and change the `127.0.0.1` in the file to that address. +To make your node accessible over the local area network, you will need to determine your machine's IP on the LAN (`ip a`), and change the `127.0.0.1` in the file to that address. -A more permissive and less secure option is to change it to `0.0.0.0`. This will allow anyone who can connect to your computer on that port to access the [API](https://lotu.sh/en+api). They will still need an auth token. +A more permissive and less secure option is to change it to `0.0.0.0`. This will allow anyone who can connect to your computer on that port to access the miner's API, though they will still need an auth token. `RemoteListenAddress` must be set to an address which other nodes on your network will be able to reach. -Next, you will need to [create an authentication token](https://lotu.sh/en+api-scripting-support#generate-a-jwt-46). All Lotus APIs require authentication tokens to ensure your processes are as secure against attackers attempting to make unauthenticated requests to them. +### Create an authentication token -### Connect the Lotus Worker +Write down the output of: -On the machine that will run `lotus-worker`, set the `MINER_API_INFO` environment variable to `TOKEN:MINER_NODE_MULTIADDR`. Where `TOKEN` is the token we created above, and `NIMER_NODE_MULTIADDR` is the `multiaddr` of the **Lotus Miner** API that was set in `config.toml`. +```sh +lotus-miner auth api-info --perm admin +``` -Once this is set, run: +The Lotus Workers will need this token to connect to the miner. + +## Connecting the Lotus Workers + +On each machine that will run the `lotus-worker` application you will need to define the following *environment variable*: + +```sh +export MINER_API_INFO::/ip4//tcp/2345` +``` + +If you are trying to use `lotus-worker` from China. You should additionally set: + +```sh +export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" +``` + + +Once that is done, you can run the Worker with: ```sh lotus-worker run ``` -If you are running multiple workers on the same host, you will need to specify the `--listen` flag and ensure each worker is on a different port. +> If you are running multiple workers on the same host, you will need to specify the `--listen` flag and ensure each worker is on a different port. -To check that the **Lotus Worker** is connected to your **Lotus Miner**, run `lotus-miner sealing workers` and check that the remote worker count has increased. +On your Lotus miner, check that the workers are correctly connected: ```sh -why@computer ~/lotus> lotus-miner sealing workers +lotus-miner sealing workers Worker 0, host computer CPU: [ ] 0 core(s) in use RAM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB @@ -69,9 +78,10 @@ Worker 1, host othercomputer GPU: GeForce RTX 2080, not used ``` -### Running locally for manually managing process priority +## Running locally for manually managing process priority You can also run the **Lotus Worker** on the same machine as your **Lotus Miner**, so you can manually manage the process priority. + To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes. You can then run the miner on your local-loopback interface; diff --git a/documentation/en/mining/managing-deals.md b/documentation/en/mining/managing-deals.md new file mode 100644 index 000000000..5f73a6a2d --- /dev/null +++ b/documentation/en/mining/managing-deals.md @@ -0,0 +1,19 @@ +# Managing deals + + +While the Lotus Miner is running as a daemon, the `lotus-miner` application can be used to manage and configure the miner: + + +```sh +lotus-miner storage-deals --help +``` + +Running the above command will show the different options related to deals. For example, `lotus-miner storage-deals set-ask` allows to set the price for storage that your miner uses to respond ask requests from clients. + +If deals are ongoing, you can check the data transfers with: + +```sh +lotus-miner data-transfers list +``` + +Make sure you explore the `lotus-miner` CLI. Every command is self-documented and takes a `--help` flag that offers specific information about it. diff --git a/documentation/en/mining/miner-setup.md b/documentation/en/mining/miner-setup.md new file mode 100644 index 000000000..cafa1e7b1 --- /dev/null +++ b/documentation/en/mining/miner-setup.md @@ -0,0 +1,241 @@ +# Miner setup + +This page will guide you through all you need to know to sucessfully run a **Lotus Miner**. Before proceeding, remember that you should be running the Lotus daemon on a fully synced chain. + +## Performance tweaks + +This is a list of performance tweaks to consider before starting the miner: + +### Building + +As [explained already](en+install-linux#native-filecoin-ffi-10) should have exported the following variables before building the Lotus applications: + +```sh +export RUSTFLAGS="-C target-cpu=native -g" +export FFI_BUILD_FROM_SOURCE=1 +``` + +### Environment + +For high performance mining, we recommend setting the following variables in your environment so that they are available when running any of the Lotus applications: + +```sh +# See https://github.com/filecoin-project/bellman +export BELLMAN_CPU_UTILIZATION=0.875 + +# See https://github.com/filecoin-project/rust-fil-proofs/ +export FIL_PROOFS_MAXIMIZE_CACHING=1 # More speed at RAM cost (1x sector-size of RAM - 32 GB). +export FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1 # precommit2 GPU acceleration +export FIL_PROOFS_USE_GPU_TREE_BUILDER=1 +``` + +IF YOU ARE RUNNING FROM CHINA: + +```sh +export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" +``` + +IF YOUR MINER RUNS IN A DIFFERENT MACHINE AS THE LOTUS DAEMON: + +```sh +export FULLNODE_API_INFO=:/ip4//tcp//http +``` + +If you will be using systemd service files to run the Lotus daemon and miner, make sure you include these variables manually in the service files. + +### Adding swap + +If you have only 128GiB of RAM, you will need to make sure your system provides at least an extra 256GiB of fast swap (preferably NVMe SSD): + +```sh +sudo fallocate -l 256G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +# show current swap spaces and take note of the current highest priority +swapon --show +# append the following line to /etc/fstab (ensure highest priority) and then reboot +# /swapfile swap swap pri=50 0 0 +sudo reboot +# check a 256GB swap file is automatically mounted and has the highest priority +swapon --show +``` + +## Creating a new BLS wallet + +You will need a BLS wallet (`t3...`) for mining. To create it, if you don't have one already, run: + +```sh +lotus wallet new bls +``` + +Next make sure to [send some funds](en+wallet) to this address so that the miner setup can be completed. + +## Initializing the miner + +> SPACE RACE: +> To participate in the Space race, please register your miner: +> +> - Visit the [faucet](http://spacerace.faucet.glif.io/) +> - Paste the address you created under REQUEST. +> - Press the Request button. + +Now that you have a miner address you can initialize the Lotus Miner: + +```sh +lotus-miner init --owner= --no-local-storage +``` + +* The `--no-local-storage` flag is used so that we configure specific locations for storage later below. +* The init process will download over 100GiB of initialization parameters to /var/tmp/filecoin-proof-parameters. Make sure there is space or set `FIL_PROOFS_PARAMETER_CACHE` to somewhere else. +* The Lotus Miner configuration folder is created at `~/.lotusminer/` or `$LOTUS_MINER_PATH` if set. + +## Reachability + +Before you start your miner, it is __very important__ to configure it so that it is reachable from any peer in the Filecoin network. For this you will need a stable public IP and edit your `~/.lotusminer/config.toml` as follows: + +```toml +... +[Libp2p] + ListenAddresses = ["/ip4/0.0.0.0/tcp/24001"] # choose a fixed port + AnnounceAddresses = ["/ip4//tcp/24001"] # important! +... +``` + +Once you start your miner, make sure you can connect to its public IP/port (you can use `telnet`, `nc` for the task...). If you have an active firewall or some sort, you may need to additionally open ports in it. + + +## Starting the miner + +You are now ready to start your Lotus miner: + +```sh +lotus-miner run +``` + +or if you are using the systemd service file: + +```sh +systemctl start lotus-miner +``` + +> __Do not proceed__ from here until you have verified that your miner not only is running, but also __reachable on its public IP address__. + +## Publishing the miner addresses + +Once the miner is up and running, publish your miner address (which you configured above) on the chain (please ensure it is dialable): + +```sh +lotus-miner actor set-addrs /ip4//tcp/24001 +``` + +## Setting locations for sealing and long-term storage + +If you used the `--no-local-storage` flag during initialization, you can now specify the disk locations for sealing (SSD recommended) and long-term storage (otherwise you can skip this): + +``` +lotus-miner storage attach --init --seal +lotus-miner storage attach --init --store +lotus-miner storage list +``` + +## Pledging sectors + +If you would like to compete for block rewards by increasing your power in the network as soon as possible, you can optionally pledge one or several sectors, depending on your storage. It can also be used to test that the sealing process works correctly. Pledging is equivalent to storing random data instead of real data obtained through storage deals. + +> Note that pledging sectors to the mainnet network makes most sense when trying to obtain a reasonable amount of total power in the network, thus obtaining real chances to mine new blocks. Otherwise it is only useful for testing purposes. + +If you decide to go ahead, then do: + +```sh +lotus-miner sectors pledge +``` + +This will write data to `$TMPDIR` so make sure that there is enough space available. + +You shoud check that your sealing job has started with: + +```sh +lotus-miner sealing jobs +``` + +This will be accommpanied by a file in `/unsealed`. + +After some minutes, you can check the sealing progress with: + +```sh +lotus-miner sectors list +# and +lotus-miner sealing workers +``` + +When sealing for the new is complete, `pSet: NO` will become `pSet: YES`. + +Once the sealing is finished, you will want to configure how long it took your miner to seal this sector and configure the miner accordingly. To find out how long it took use: + +``` +lotus-miner sectors status --log 0 +``` + +Once you know, you can edit the Miner's `~/.lotusminer/config.toml` accordingly: + +``` +... +[Dealmaking] +... + ExpectedSealDuration = "12h0m0s" # The time it took your miner +``` + +You can also take the chance to edit other values, such as `WaitForDealsDelay` which specifies the delay between accepting the first deal and sealing, allowing to place multiple deals in the same sector. + +Once you are done editing the configuration, [restart your miner](en+update). + +If you wish to be able to re-use a pledged sector for real storage deals before the pledged period of 6 months ends, you will need to mark them for upgrade: + +```sh +lotus-miner sectors mark-for-upgrade +``` + +The sector should become inactive within 24 hours. From that point, the pledged storage can be re-used to store real data associated with real storage deals. + +## Separate address for windowPoSt messages + +WindowPoSt is the mechanism through which storage is verified in Filecoin. It requires miners to submit proofs for all sectors every 24h, which require sending messages to the chain. + +Because many other mining related actions require sending messages to the chain, and not all of those are "high value", it may be desirable to use a separate account to send PoSt messages from. This allows for setting lower GasFeeCaps on the lower value messages without creating head-of-line blocking problems for the PoSt messages in congested chain conditions + +To set this up, first create a new account, and send it some funds for gas fees: + +```sh +lotus wallet new bls +t3defg... + +lotus send t3defg... 100 +``` + +Next add the control address: + +```sh +lotus-miner actor control set --really-do-it t3defg... +Add t3defg... +Message CID: bafy2.. +``` + +Wait for the message to land on chain: + +```sh +lotus state wait-msg bafy2.. +... +Exit Code: 0 +... +``` + +Finally, check the miner control address list to make sure the address was correctly setup: + +```sh +lotus-miner actor control list +name ID key use balance +owner t01111 t3abcd... other 300 FIL +worker t01111 t3abcd... other 300 FIL +control-0 t02222 t3defg... post 100 FIL +``` diff --git a/documentation/en/mining-troubleshooting.md b/documentation/en/mining/mining-troubleshooting.md similarity index 90% rename from documentation/en/mining-troubleshooting.md rename to documentation/en/mining/mining-troubleshooting.md index 5aaf9f6ef..758929075 100644 --- a/documentation/en/mining-troubleshooting.md +++ b/documentation/en/mining/mining-troubleshooting.md @@ -25,7 +25,7 @@ lotus-miner info # WARN main lotus-storage-miner/main.go:73 failed to get api endpoint: (/Users/myrmidon/.lotusminer) %!w(*errors.errorString=&{API not running (no endpoint)}): ``` -If you see this, that means your **Lotus Miner** isn't ready yet. You need to finish [syncing the chain](https://lotu.sh/en+join-testnet). +If you see this, that means your **Lotus Miner** isn't ready yet. You need to finish [syncing the chain](en+setup#waiting-to-sync-370). ## Error: Your computer may not be fast enough @@ -57,10 +57,3 @@ make bench This process uses a fair amount of GPU, and generally takes ~4 minutes to complete. If you do not see any activity in nvtop from lotus during the entire process, it is likely something is misconfigured with your GPU. -## Checking Sync Progress - -You can use this command to check how far behind you are on syncing: - -```sh -date -d @$(./lotus chain getblock $(./lotus chain head) | jq .Timestamp) -``` diff --git a/documentation/en/mining/mining.md b/documentation/en/mining/mining.md new file mode 100644 index 000000000..b1b944c6e --- /dev/null +++ b/documentation/en/mining/mining.md @@ -0,0 +1,8 @@ +# Storage Mining + +This section of the documentation explains how to do storage mining with Lotus. Please note that not everyone can do storage mining, and that you should not attempt it on on networks where sector sizes are 32GB+ unless you meet the [hardware requirements](en+install#hardware-requirements-1). + +From this point we assume that you have setup and are running the [Lotus Node](en+setup), that it has fully synced the Filecoin chain and that you are familiar with how to interact with it using the `lotus` command-line interface. + +In order to perform storage mining, apart from the Lotus daemon, you will be additionally interacting with the `lotus-miner` and potentially the `lotus-worker` applications (which you should have [installed](en+install-linux) along the `lotus` application already). + diff --git a/documentation/en/retrieving-data.md b/documentation/en/retrieving-data.md deleted file mode 100644 index 7cb0e31be..000000000 --- a/documentation/en/retrieving-data.md +++ /dev/null @@ -1,27 +0,0 @@ -# Retrieving Data - -> There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can. - -Here are the operations you can perform after you have stored and sealed a **Data CID** with the **Lotus Miner** in the network. - -If you would like to learn how to store a **Data CID** on a miner, read the instructions [here](https://lotu.sh/en+storing-data). - -## Find by Data CID - -```sh -lotus client find -# LOCAL -# RETRIEVAL @-- -``` - -## Retrieve by Data CID - -All fields are required. - -```sh -lotus client retrieve -``` - -If the outfile does not exist it will be created in the Lotus repository directory. - -This command will initiate a **retrieval deal** and write the data to your computer. This process may take 2 to 10 minutes. diff --git a/documentation/en/setting-a-static-port.md b/documentation/en/setting-a-static-port.md deleted file mode 100644 index 97ac6528e..000000000 --- a/documentation/en/setting-a-static-port.md +++ /dev/null @@ -1,54 +0,0 @@ -# Static Ports - -Depending on how your network is set up, you may need to set a static port to successfully connect to peers to perform storage deals with your **Lotus Miner**. - -## Setup - -To change the random **swarm port**, you may edit the `config.toml` file located under `$LOTUS_MINER_PATH`. The default location of this file is `$HOME/.lotusminer`. - -To change the port to `1347`: - -```sh -[Libp2p] - ListenAddresses = ["/ip4/0.0.0.0/tcp/1347", "/ip6/::/tcp/1347"] -``` - -After changing the port value, restart your **daemon**. - -## Announce Addresses - -If the **swarm port** is port-forwarded from another address, it is possible to control what addresses -are announced to the network. - -```sh -[Libp2p] - AnnounceAddresses = ["/ip4//tcp/1347"] -``` - -If non-empty, this array specifies the swarm addresses to announce to the network. If empty, the daemon will announce inferred swarm addresses. - -Similarly, it is possible to set `NoAnnounceAddresses` with an array of addresses to not announce to the network. - -## Ubuntu's Uncomplicated Firewall - -Open firewall manually: - -```sh -ufw allow 1347/tcp -``` - -Or open and modify the profile located at `/etc/ufw/applications.d/lotus-daemon`: - -```sh -[Lotus Daemon] -title=Lotus Daemon -description=Lotus Daemon firewall rules -ports=1347/tcp -``` - -Then run these commands: - -```sh -ufw update lotus-daemon -ufw allow lotus-daemon -``` diff --git a/documentation/en/storing-ipfs-integration.md b/documentation/en/store/adding-from-ipfs.md similarity index 79% rename from documentation/en/storing-ipfs-integration.md rename to documentation/en/store/adding-from-ipfs.md index 041364380..2f6b097cc 100644 --- a/documentation/en/storing-ipfs-integration.md +++ b/documentation/en/store/adding-from-ipfs.md @@ -1,10 +1,10 @@ -# IPFS Integration +# Adding data from IPFS Lotus supports making deals with data stored in IPFS, without having to re-import it into lotus. To enable this integration, you need to have an IPFS daemon running in the background. -Then, open up `~/.lotus/config.toml` (or if you manually set `LOTUS_PATH`, look under that directory) -and look for the Client field, and set `UseIpfs` to `true`. + +Then, open up `~/.lotus/config.toml` (or if you manually set `LOTUS_PATH`, look under that directory) and look for the Client field, and set `UseIpfs` to `true`. ```toml [Client] diff --git a/documentation/en/store/making-deals.md b/documentation/en/store/making-deals.md new file mode 100644 index 000000000..ca3a47182 --- /dev/null +++ b/documentation/en/store/making-deals.md @@ -0,0 +1,71 @@ +# Making storage deals + +## Adding a file to Lotus + +Before sending data to a Filecoin miner for storage, the data needs to be correctly formatted and packed. This can be achieved by locally importing the data into Lotus with: + +```sh +lotus client import ./your-example-file.txt +``` + +Upon success, this command will return a **Data CID**. This is a very important piece of information, as it will be used to make deals to both store and retrieve the data in the future. + +You can list the data CIDs of the files you locally imported with: + +```sh +lotus client local +``` + +## Storing data in the network + +To store data in the network you will need to: + +* Find a Filecoin miner willing to store it +* Make a deal with the miner agreeing on the price to pay and the duration for which the data should be stored. + +You can obtain a list of all miners in the network with: + +```sh +lotus state list-miners +t0xxxx +t0xxxy +t0xxxz +... +``` + +This will print a list of miner IDs. In order to ask for the terms offered by a particular miner, you can then run: + +```sh +lotus client query-ask +``` + +If you are satisfied with the terms, you can proceed to propose a deal to the miner, using the **Data CID** that you obtained during the import step: + + +```sh +lotus client deal +``` + +This command will interactively ask you for the CID, miner ID and duration in days for the deal. You can also call it with arguments: + +```sh +lotus client deal +``` + +where the `duration` is expressed in blocks (1 block is equivalent to 30s). + +## Checking the status of the deals + +You can list deals with: + +```sh +lotus client list-deals +``` + +Among other things, this will give you information about the current state on your deals, whether they have been published on chain (by the miners) and whether the miners have been slashed for not honoring them. + +For a deal to succeed, the miner needs to be correctly configured and running, accept the deal and *seal* the file correctly. Otherwise, the deal will appear in error state. + +You can make deals with multiple miners for the same data. + +Once a deal is sucessful and the data is *sealed*, it can be [retrieved](en+retrieving). diff --git a/documentation/en/store/retrieve.md b/documentation/en/store/retrieve.md new file mode 100644 index 000000000..1e8db65af --- /dev/null +++ b/documentation/en/store/retrieve.md @@ -0,0 +1,27 @@ +# Retrieving Data + +Once data has been succesfully [stored](en+making-deals) and sealed by a Filecoin miner, it can be retrieved. + +In order to do this we will need to create a **retrieval deal**. + +## Finding data by CID + +In order to retrieve some data you will need the **Data CID** that was used to create the storage deal. + +You can find who is storing the data by running: + +```sh +lotus client find +``` + +## Making a retrieval deal + +You can then make a retrieval deal with: + +```sh +lotus client retrieve +``` + +This commands take other optional flags (check `--help`). + +If the outfile does not exist it will be created in the Lotus repository directory. This process may take 2 to 10 minutes. diff --git a/documentation/en/storing-data-troubleshooting.md b/documentation/en/store/storage-troubleshooting.md similarity index 51% rename from documentation/en/storing-data-troubleshooting.md rename to documentation/en/store/storage-troubleshooting.md index c8a0254fa..7087ec3d0 100644 --- a/documentation/en/storing-data-troubleshooting.md +++ b/documentation/en/store/storage-troubleshooting.md @@ -2,11 +2,11 @@ ## Error: Routing: not found -```sh +``` WARN main lotus/main.go:72 routing: not found ``` -- This miner is offline. +This error means that the miner is offline. ## Error: Failed to start deal @@ -14,14 +14,17 @@ WARN main lotus/main.go:72 routing: not found WARN main lotus/main.go:72 failed to start deal: computing commP failed: generating CommP: Piece must be at least 127 bytes ``` -- There is a minimum file size of 127 bytes. +This error means that there is a minimum file size of 127 bytes. ## Error: 0kb file response during retrieval -In order to retrieve a file, it must be sealed. Miners can check sealing progress with this command: +This means that the file to be retrieved may have not yet been sealed and is thus, not retrievable yet. + +Miners can check sealing progress with this command: ```sh lotus-miner sectors list ``` -When sealing is complete, `pSet: NO` will become `pSet: YES`. From now on the **Data CID** is [retrievable](https://lotu.sh/en+retrieving-data) from the **Lotus Miner**. +When sealing is complete, `pSet: NO` will become `pSet: YES`. + diff --git a/documentation/en/store/store.md b/documentation/en/store/store.md new file mode 100644 index 000000000..205bd0e23 --- /dev/null +++ b/documentation/en/store/store.md @@ -0,0 +1,11 @@ +# Storing and retrieving data + +Lotus enables you to store any data on the Filecoin network and retrieve it later. This is achieved by making *deals* with miners. + +A *storage deal* specifies that a miner should store ceratin data for a previously agreed period and price. + +Once a deal is made, the data is then sent to the miners, which regularly proves that it is storing it. If they fail to do so, the miner is penalized (slashed). + +The data can be retrieved with a *retrieval deal*. + +This section explains how to use Lotus to [store](en+making-deals) and [retrieve](en+retrieving) data from the Filecoin network. diff --git a/documentation/en/storing-data.md b/documentation/en/storing-data.md deleted file mode 100644 index 67d2b1533..000000000 --- a/documentation/en/storing-data.md +++ /dev/null @@ -1,62 +0,0 @@ -# Storing Data - -> There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can. - -Here are instructions for how to store data on the **Lotus Testnet**. - -## Adding a file locally - -Adding a file locally allows you to make miner deals on the **Lotus Testnet**. - -```sh -lotus client import ./your-example-file.txt -``` - -Upon success, this command will return a **Data CID**. - -## List your local files - -The command to see a list of files by `CID`, `name`, `size` in bytes, and `status`: - -```sh -lotus client local -``` - -An example of the output: - -```sh -bafkreierupr5ioxn4obwly4i2a5cd2rwxqi6kwmcyyylifxjsmos7hrgpe Development/sample-1.txt 2332 ok -bafkreieuk7h4zs5alzpdyhlph4lxkefowvwdho3a3pml6j7dam5mipzaii Development/sample-2.txt 30618 ok -``` - -## Make a Miner Deal on Lotus Testnet - -Get a list of all miners that can store data: - -```sh -lotus state list-miners -``` - -Get the requirements of a miner you wish to store data with: - -```sh -lotus client query-ask -``` - -Store a **Data CID** with a miner: - -```sh -lotus client deal -``` - -Check the status of a deal: - -```sh -lotus client list-deals -``` - -- The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 25 seconds. - -Upon success, this command will return a **Deal CID**. - -The miner will need to **seal** the file before it can be retrieved. If the **Lotus Miner** is not running on a machine designed for sealing, the process will take a very long time. diff --git a/documentation/en/dev/WIP-arch-complementary-notes.md b/documentation/en/unclassified/WIP-arch-complementary-notes.md similarity index 100% rename from documentation/en/dev/WIP-arch-complementary-notes.md rename to documentation/en/unclassified/WIP-arch-complementary-notes.md diff --git a/documentation/en/block-validation.md b/documentation/en/unclassified/block-validation.md similarity index 100% rename from documentation/en/block-validation.md rename to documentation/en/unclassified/block-validation.md diff --git a/documentation/en/dev/create-miner.md b/documentation/en/unclassified/create-miner.md similarity index 100% rename from documentation/en/dev/create-miner.md rename to documentation/en/unclassified/create-miner.md diff --git a/documentation/en/dev-tools-pond-ui.md b/documentation/en/unclassified/dev-tools-pond-ui.md similarity index 100% rename from documentation/en/dev-tools-pond-ui.md rename to documentation/en/unclassified/dev-tools-pond-ui.md diff --git a/documentation/en/sealing-procs.md b/documentation/en/unclassified/sealing-procs.md similarity index 100% rename from documentation/en/sealing-procs.md rename to documentation/en/unclassified/sealing-procs.md diff --git a/documentation/en/updating-lotus.md b/documentation/en/updating-lotus.md deleted file mode 100644 index 862cea136..000000000 --- a/documentation/en/updating-lotus.md +++ /dev/null @@ -1,14 +0,0 @@ -# Updating Lotus - -If you installed Lotus on your machine, you can upgrade to the latest version by doing the following: - -```sh -# get the latest -git pull origin master - -# clean and remake the binaries -make clean && make build - -# instal binaries in correct location -make install # or sudo make install if necessary -``` From a153e1d5862b67a031fe7ee7306a7e430c517187 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 31 Aug 2020 21:18:50 +0200 Subject: [PATCH 04/80] Fix #2334: Specify seal options to disable with co-located storage worker --- documentation/en/mining/lotus-seal-worker.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/documentation/en/mining/lotus-seal-worker.md b/documentation/en/mining/lotus-seal-worker.md index 62dde9bfb..47e201ca5 100644 --- a/documentation/en/mining/lotus-seal-worker.md +++ b/documentation/en/mining/lotus-seal-worker.md @@ -82,7 +82,15 @@ Worker 1, host othercomputer You can also run the **Lotus Worker** on the same machine as your **Lotus Miner**, so you can manually manage the process priority. -To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes. +To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes: + +```toml +[Storage] + AllowPreCommit1 = false + AllowPreCommit2 = false + AllowCommit = false + AllowUnseal = false +``` You can then run the miner on your local-loopback interface; From af38c902f88d5c7be13f24eca4e8d19b395e963f Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 1 Sep 2020 15:12:07 +0200 Subject: [PATCH 05/80] Fix architecture entry --- documentation/en/.library.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 87c7353c1..59cc01e29 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -193,7 +193,7 @@ { "title": "Lotus Architecture (WIP)", "slug": "en+arch", - "github": "en/architectiure/architecture.md", + "github": "en/architecture/architecture.md", "value": null, "posts": [ { From bd0c6a4cccd5a5e21ae3f4939cf9d8822ea9ca37 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 3 Sep 2020 16:35:55 +0200 Subject: [PATCH 06/80] Fix filename --- documentation/en/building/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/en/building/api.md b/documentation/en/building/api.md index 626193ee2..3a2c2902b 100644 --- a/documentation/en/building/api.md +++ b/documentation/en/building/api.md @@ -19,7 +19,7 @@ Lotus uses its own Go library implementation of [JSON-RPC](https://github.com/fi ## cURL example -To demonstrate making an API request, we will take the method `ChainHead` from [api/api.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). +To demonstrate making an API request, we will take the method `ChainHead` from [api/api_full.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). ```go ChainHead(context.Context) (*types.TipSet, error) From e53aee26a3d1b440729212fde7c77b8c06bf9a14 Mon Sep 17 00:00:00 2001 From: austinabell Date: Tue, 15 Sep 2020 18:36:06 -0400 Subject: [PATCH 07/80] Make state transition in validation async --- chain/sync.go | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index f7530f556..e23284134 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -788,31 +788,35 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er b.Header.ParentWeight, pweight) } - // Stuff that needs stateroot / worker address - stateroot, precp, err := syncer.sm.TipSetState(ctx, baseTs) - if err != nil { - return xerrors.Errorf("get tipsetstate(%d, %s) failed: %w", h.Height, h.Parents, err) - } - - if stateroot != h.ParentStateRoot { - msgs, err := syncer.store.MessagesForTipset(baseTs) + stateRootCheck := async.Err(func() error { + stateroot, precp, err := syncer.sm.TipSetState(ctx, baseTs) if err != nil { - log.Error("failed to load messages for tipset during tipset state mismatch error: ", err) - } else { - log.Warn("Messages for tipset with mismatching state:") - for i, m := range msgs { - mm := m.VMMessage() - log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params) - } + return xerrors.Errorf("get tipsetstate(%d, %s) failed: %w", h.Height, h.Parents, err) } - return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot) - } + if stateroot != h.ParentStateRoot { + msgs, err := syncer.store.MessagesForTipset(baseTs) + if err != nil { + log.Error("failed to load messages for tipset during tipset state mismatch error: ", err) + } else { + log.Warn("Messages for tipset with mismatching state:") + for i, m := range msgs { + mm := m.VMMessage() + log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params) + } + } - if precp != h.ParentMessageReceipts { - return xerrors.Errorf("parent receipts root did not match computed value (%s != %s)", precp, h.ParentMessageReceipts) - } + return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot) + } + if precp != h.ParentMessageReceipts { + return xerrors.Errorf("parent receipts root did not match computed value (%s != %s)", precp, h.ParentMessageReceipts) + } + + return nil + }) + + // Stuff that needs worker address waddr, err := stmgr.GetMinerWorkerRaw(ctx, syncer.sm, lbst, h.Miner) if err != nil { return xerrors.Errorf("GetMinerWorkerRaw failed: %w", err) @@ -933,6 +937,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er winnerCheck, msgsCheck, baseFeeCheck, + stateRootCheck, } var merr error From a4fd356fcbc20f38657c128ceb291c2db42f150a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:35:19 +0200 Subject: [PATCH 08/80] Delete most docs. Update about page with links to docs.filecoin.io --- documentation/en/.glossary.json | 147 +- documentation/en/.library.json | 202 +- documentation/en/about.md | 19 +- documentation/en/building/api-methods.md | 4567 ----------------- .../en/building/api-troubleshooting.md | 36 - documentation/en/building/api.md | 38 - documentation/en/building/building.md | 5 - documentation/en/building/jaeger-tracing.md | 26 - documentation/en/building/local-devnet.md | 54 - documentation/en/building/payment-channels.md | 111 - documentation/en/building/remote-api.md | 69 - .../en/getting-started/getting-started.md | 3 - .../getting-started/setup-troubleshooting.md | 57 - documentation/en/getting-started/setup.md | 169 - documentation/en/getting-started/wallet.md | 58 - .../en/installation/install-linux.md | 129 - .../en/installation/install-macos.md | 62 - documentation/en/installation/installation.md | 39 - documentation/en/installation/update.md | 72 - documentation/en/mining/gpus.md | 17 - documentation/en/mining/lotus-seal-worker.md | 99 - documentation/en/mining/managing-deals.md | 19 - documentation/en/mining/miner-setup.md | 241 - .../en/mining/mining-troubleshooting.md | 59 - documentation/en/mining/mining.md | 8 - documentation/en/store/adding-from-ipfs.md | 20 - documentation/en/store/making-deals.md | 71 - documentation/en/store/retrieve.md | 27 - .../en/store/storage-troubleshooting.md | 30 - documentation/en/store/store.md | 11 - 30 files changed, 14 insertions(+), 6451 deletions(-) delete mode 100644 documentation/en/building/api-methods.md delete mode 100644 documentation/en/building/api-troubleshooting.md delete mode 100644 documentation/en/building/api.md delete mode 100644 documentation/en/building/building.md delete mode 100644 documentation/en/building/jaeger-tracing.md delete mode 100644 documentation/en/building/local-devnet.md delete mode 100644 documentation/en/building/payment-channels.md delete mode 100644 documentation/en/building/remote-api.md delete mode 100644 documentation/en/getting-started/getting-started.md delete mode 100644 documentation/en/getting-started/setup-troubleshooting.md delete mode 100644 documentation/en/getting-started/setup.md delete mode 100644 documentation/en/getting-started/wallet.md delete mode 100644 documentation/en/installation/install-linux.md delete mode 100644 documentation/en/installation/install-macos.md delete mode 100644 documentation/en/installation/installation.md delete mode 100644 documentation/en/installation/update.md delete mode 100644 documentation/en/mining/gpus.md delete mode 100644 documentation/en/mining/lotus-seal-worker.md delete mode 100644 documentation/en/mining/managing-deals.md delete mode 100644 documentation/en/mining/miner-setup.md delete mode 100644 documentation/en/mining/mining-troubleshooting.md delete mode 100644 documentation/en/mining/mining.md delete mode 100644 documentation/en/store/adding-from-ipfs.md delete mode 100644 documentation/en/store/making-deals.md delete mode 100644 documentation/en/store/retrieve.md delete mode 100644 documentation/en/store/storage-troubleshooting.md delete mode 100644 documentation/en/store/store.md diff --git a/documentation/en/.glossary.json b/documentation/en/.glossary.json index e8a9e0846..0967ef424 100644 --- a/documentation/en/.glossary.json +++ b/documentation/en/.glossary.json @@ -1,146 +1 @@ -{ - "bellman": { - "title": "Bellman", - "value": "Bellman is a rust crate for building zk-SNARK circuits. It provides circuit traits and primitive structures, as well as basic gadget implementations such as booleans and number abstractions." - }, - "nvme": { - "title": "NVMe", - "value": "(non-volatile memory express) is a host controller interface and storage protocol created to accelerate the transfer of data between enterprise and client systems and solid-state drives (SSDs) over a computer's high-speed Peripheral Component Interconnect Express (PCIe) bus." - }, - "multiaddr": { - "title": "Multiaddr", - "value": "Multiaddr is a format for encoding addresses from various well-established network protocols. It is useful to write applications that future-proof their use of addresses, and allow multiple transport protocols and addresses to coexist." - }, - "attofil": { - "title": "attoFIL", - "value": "AttoFIL is a word used to describe 10^-18 FIL. The word atto comes from the Norwegian and Danish term: atten eighteen." - }, - "fil": { - "title": "FIL", - "value": "A ticker symbol is an abbreviation used to uniquely identify Filecoin when it is used in a wallet exchange or a cryptocurrency exchange." - }, - "epost": { - "title": "Election Proof-of-Spacetime", - "value": "Election Proof-of-Spacetime couples the Proof-of-Spacetime process with block production, meaning that in order to produce a block, the miner must produce a valid Proof-of-Spacetime proof (snark output)." - }, - "jwt": { - "title": "JWT", - "value": "JSON Web Tokens are an open, industry standard RFC 7519 method for representing claims securely between two parties." - }, - "json-rpc": { - "title": "JSON-RPC", - "value": "JSON-RPC is a remote procedure call protocol encoded in JSON. It is a very simple protocol (and very similar to XML-RPC), defining only a few data types and commands." - }, - "bls-address": { - "title": "BLS Signature (Address)", - "value": "A Boneh–Lynn–Shacham (BLS) signature is a digital signature scheme that allows a user to determine the authenticity of a signer, and is a commonly used signature scheme in the Filecoin Distributed Storage Network." - }, - "faucet": { - "title": "Filecoin Test Faucet", - "value": "A webpage where you can get free test Filecoin to participate in the Testnet." - }, - "chain": { - "title": "Chain", - "value": "The Filecoin Blockchain is a distributed virtual machine that achieves consensus, processes messages, accounts for storage, and maintains security in the Filecoin Protocol. It is the main interface linking various actors in the Filecoin system." - }, - "miner-power": { - "title": "Miner Power", - "value": "Miner storage in relation to network storage, tracked in the power table." - }, - "sector": { - "title": "Sector", - "value": "A fixed-size block of data of SECTOR_SIZE bytes which generally contains client's data." - }, - "sealing": { - "title": "Sealing", - "value": "A slow encoding process that returns commitments and proofs for data being stored in a sector." - }, - "seal": { - "title": "Seal", - "value": "A slow encoding process that returns commitments and proofs for data being stored in a sector." - }, - "posts": { - "title": "Proof-of-Spacetime(s)", - "value": "Filecoin is a protocol token whose blockchain runs on a novel proof, called Proof-of-Spacetime, where blocks are created by miners that are storing data." - }, - "filecoin-testnet": { - "title": "Filecoin Testnet", - "value": "Until we launch, we are making lots of changes to Lotus. The Testnet is expected to bring a few significant fixes/improvements. During Testnet, you can retrieve test filecoin from our network faucet to use as collateral to start mining. Test filecoin do not have any value – the official filecoin tokens will not be released until Mainnet launch." - }, - "filecoin-decentralized-storage-market": { - "title": "Filecoin Decentralized Storage Market", - "value": "Storage Market subsystem is the data entry point into the network. Miners only earn power from data stored in a storage deal and all deals live on the Filecoin network." - }, - "filecoin-proof-parameters": { - "title": "Filecoin Proof Parameters", - "value": "The proving algorithms rely on a large binary parameter file." - }, - "lotus-devnet": { - "title": "DevNet", - "value": "On the DevNets, you can store data as a storage client and also try how Filecoin mining works. The devnets are an important development tool for those who anticipate building applications on top of the Filecoin protocol or storing data on the decentralized storage market. " - }, - "filecoin-distributed-storage-network": { - "title": "Filecoin Distributed Storage Network", - "value": "Filecoin is a distributed storage network based on a blockchain mechanism. Filecoin miners can elect to provide storage capacity for the network, and thereby earn units of the Filecoin cryptocurrency (FIL) by periodically producing cryptographic proofs that certify that they are providing the capacity specified." - }, - "lotus-node": { - "title": "Lotus Node", - "value": "The Lotus Node is full of capabilities. It runs the Blockchain system, makes retrieval deals, does data transfer, supports block producer logic, and syncs and validates the chain." - }, - "block-rewards": { - "title": "Block Reward", - "value": "Over the entire lifetime of the protocol, 1,400,000,000 FIL (TotalIssuance) will be given out to miners. The rate at which the funds are given out is set to halve every six years, smoothly (not a fixed jump like in Bitcoin)." - }, - "block-producer-miner": { - "title": "Miner (Block Producer)", - "value": "The Block Producer Miner's logic. It currently shares an interface and process with the Lotus Node. A Block Producer chooses which messages to include in a block and is rewarded according to each message’s gas price and consumption, forming a market." - }, - "lotus-miner": { - "title": "Miner (lotus-miner)", - "value": "The Miner's logic. It has its own dedicated process. Contributes to the network through Sector commitments and Proofs of Spacetime to prove that it is storing the sectors it has commited to." - }, - "swarm-port": { - "title": "Swarm Port (Libp2p)", - "value": "The LibP2P Swarm manages groups of connections to peers, handles incoming and outgoing streams, and is part of the miners implementation. The port value is part of the Host interface." - }, - "daemon": { - "title": "Lotus Daemon", - "value": "A Daemon is a program that runs as a background process. A Daemon in the context of the Filecoin Distributed Storage Network may enable applications to communicate with peers, handle protocols, participate in pubsub, and interact with a distributed hash table (DHT)." - }, - "storage-deal": { - "title": "Storage deal", - "value": "One of the two types of deals in Filecoin markets. Storage deals are recorded on the blockchain and enforced by the protocol." - }, - "retrieval-deal": { - "title": "Retrieval deal", - "value": "One of the two types of deals in Filecoin markets. Retrieval deals are off chain and enabled by micropayment channel by transacting parties." - }, - "deal-cid": { - "title": "Deal CID", - "value": "CID is a format for referencing content in distributed information systems, it is a way to store information so it can be retrieved based on its content, not its location. DealCID specifically is used in storage deals." - }, - "data-cid": { - "title": "Data CID", - "value": "CID is a format for referencing content in distributed information systems, it is a way to store information so it can be retrieved based on its content, not its location. DataCID specifically is used to represent the file that is stored in the Filecoin Distributed Storage Network." - }, - "cid": { - "title": "CID", - "value": "A CID is a self-describing content-addressed identifier. It uses cryptographic hashes to achieve content addressing. It uses several multiformats to achieve flexible self-description, namely multihash for hashes, multicodec for data content types, and multibase to encode the CID itself into strings." - }, - "total-network-power": { - "title": "Total Network Power", - "value": "A reference to all the Power Tables for every subchain, accounting for each Lotus Miner on chain." - }, - "chain-block-height": { - "title": "Chain Block Height", - "value": "Chain block height is defined as the number of blocks in the chain between any given block and the very first block in the blockchain." - }, - "block-height": { - "title": "Block Height", - "value": "Height of the Merkle Tree of a sector. A sector is a contiguous array of bytes that a miner puts together, seals, and performs Proofs of Spacetime on." - }, - "blocktime": { - "title": "Blocktime", - "value": "The time it takes for a Block to propagate to the whole network." - } -} +{} diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 59cc01e29..e31f09950 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -2,194 +2,11 @@ "posts": [ { "title": "About Lotus", - "slug": "en+lotus", + "slug": "", "github": "en/about.md", "value": null, "posts": [] }, - { - "title": "Installation", - "slug": "en+install", - "github": "en/installation/installation.md", - "value": null, - "posts": [ - { - "title": "Linux installation", - "slug": "en+install-linux", - "github": "en/installation/install-linux.md", - "value": null - }, - { - "title": "MacOS installation", - "slug": "en+install-macos", - "github": "en/installation/install-macos.md", - "value": null - }, - { - "title": "Updating Lotus", - "slug": "en+update", - "github": "en/installation/update.md", - "value": null - } - ] - }, - { - "title": "Getting started", - "slug": "en+getting-started", - "github": "en/getting-started/getting-started.md", - "value": null, - "posts": [ - { - "title": "Setting up Lotus", - "slug": "en+setup", - "github": "en/getting-started/setup.md", - "value": null - }, - { - - "title": "Obtaining and sending FIL", - "slug": "en+wallet", - "github": "en/getting-started/wallet.md", - "value": null - }, - { - "title": "Setup troubleshooting", - "slug": "en+setup-troubleshooting", - "github": "en/getting-started/setup-troubleshooting.md", - "value": null - } - ] - }, - { - "title": "Storing and retrieving data", - "slug": "en+store", - "github": "en/store/store.md", - "value": null, - "posts": [ - { - "title": "Making storage deals", - "slug": "en+making-deals", - "github": "en/store/making-deals.md", - "value": null - }, - { - "title": "Adding data from IPFS", - "slug": "en+adding-from-ipfs", - "github": "en/store/adding-from-ipfs.md", - "value": null - }, - { - "title": "Retrieving data", - "slug": "en+retriving", - "github": "en/store/retrieve.md", - "value": null - }, - { - "title": "Storage Troubleshooting", - "slug": "en+storage-troubleshooting", - "github": "en/store/storage-troubleshooting.md", - "value": null - } - ] - }, - { - "title": "Storage mining", - "slug": "en+mining", - "github": "en/mining/mining.md", - "value": null, - "posts": [ - { - "title": "Miner setup", - "slug": "en+miner-setup", - "github": "en/mining/miner-setup.md", - "value": null - }, - { - "title": "Managing deals", - "slug": "en+managing-deals", - "github": "en/mining/managing-deals.md", - "value": null - }, - { - "title": "Lotus Worker", - "slug": "en+lotus-worker", - "github": "en/mining/lotus-seal-worker.md", - "value": null - }, - { - "title": "Benchmarking GPUs", - "slug": "en+gpus", - "github": "en/mining/gpus.md", - "value": null - }, - { - "title": "Mining Troubleshooting", - "slug": "en+mining-troubleshooting", - "github": "en/mining/mining-troubleshooting.md", - "value": null - } - ] - }, - { - "title": "Building", - "slug": "en+building", - "github": "en/building/building.md", - "value": null, - "posts": [ - { - "title": "Setting up remote API access", - "slug": "en+remote-api", - "github": "en/building/remote-api.md", - "value": null, - "posts": [] - }, - { - "title": "API endpoints and methods", - "slug": "en+api", - "github": "en/building/api.md", - "value": null, - "posts": [] - }, - { - "title": "API Reference", - "slug": "en+api-methods", - "github": "en/building/api-methods.md", - "value": null, - "posts": [] - }, - - { - "title": "Payment Channels", - "slug": "en+payment-channels", - "github": "en/building/payment-channels.md", - "value": null, - "posts": [] - }, - - { - "title": "Running a local devnet", - "slug": "en+local-devnet", - "github": "en/building/local-devnet.md", - "value": null, - "posts": [] - }, - { - "title": "Jaeger Tracing", - "slug": "en+jaeger-tracing", - "github": "en/building/jaeger-tracing.md", - "value": null, - "posts": [] - }, - - { - "title": "API Troubleshooting", - "slug": "en+api-troubleshooting", - "github": "en/building/api-troubleshooting.md", - "value": null, - "posts": [] - } - ] - }, { "title": "Lotus Architecture (WIP)", "slug": "en+arch", @@ -203,23 +20,6 @@ "value": null } ] - }, - { - "title": "FAQs", - "slug": "en+faqs", - "github": "en/faqs.md", - "value": null, - "posts": [] - }, - { - "title": "Glossary", - "slug": "en+glossary", - "github": "en/.glossary.json", - "value": null, - "custom": { - "glossary": true - }, - "posts": [] } ] } diff --git a/documentation/en/about.md b/documentation/en/about.md index ee8536ac9..f2051e00b 100644 --- a/documentation/en/about.md +++ b/documentation/en/about.md @@ -2,13 +2,18 @@ Lotus is an implementation of the **Filecoin Distributed Storage Network**. -The **Lotus Node** (and the mining applications) can be built to join any of the [Filecoin networks](https://docs.filecoin.io/how-to/networks/). +It is written in Go and provides a suite of command-line applications: -For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://filecoin-project.github.io/specs/). +- Lotus Node (`lotus`): a Filecoin Node: validates network transactions, manages a FIL wallet, can perform storage and retrieval deals. +- Lotus Miner (`lotus-miner`): a Filecoin miner. See the the respective Lotus Miner section in the Mine documentation. +- Lotus Worker (`lotus-worker`): a worker that assists miners to perform mining-related tasks. See its respective guide for more information. -## What can I learn here? +The [Lotus user documentation](https://docs.filecoin.io/get-started/lotus) is part of the [Filecoin documentation site](https://docs.filecoin.io): + +* To install and get started with Lotus, visit the [Get Started section](https://docs.filecoin.io/get-started/lotus). +* Information about how to perform deals on the Filecoin network using Lotus can be found in the [Store section](https://docs.filecoin.io/store/lotus). +* Miners looking to provide storage to the Network can find the latest guides in the [Mine section](https://docs.filecoin.io/mine/lotus). +* Developers and integrators that wish to use the Lotus APIs can start in the [Build section](https://docs.filecoin.io/mine/lotus). + +For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://spec.filecoin.io/). -* How to [install](en+installation) and [setup](en+setup) the Lotus software -* How to [store data on the Filecoin network](en+store) -* How to [setup a high performance FIL miner](en+miner-setup) -* How to [configure and access Lotus APIs](en+remote-api) diff --git a/documentation/en/building/api-methods.md b/documentation/en/building/api-methods.md deleted file mode 100644 index 2f3164bb7..000000000 --- a/documentation/en/building/api-methods.md +++ /dev/null @@ -1,4567 +0,0 @@ -# Groups -* [](#) - * [Closing](#Closing) - * [Shutdown](#Shutdown) - * [Version](#Version) -* [Auth](#Auth) - * [AuthNew](#AuthNew) - * [AuthVerify](#AuthVerify) -* [Beacon](#Beacon) - * [BeaconGetEntry](#BeaconGetEntry) -* [Chain](#Chain) - * [ChainExport](#ChainExport) - * [ChainGetBlock](#ChainGetBlock) - * [ChainGetBlockMessages](#ChainGetBlockMessages) - * [ChainGetGenesis](#ChainGetGenesis) - * [ChainGetMessage](#ChainGetMessage) - * [ChainGetNode](#ChainGetNode) - * [ChainGetParentMessages](#ChainGetParentMessages) - * [ChainGetParentReceipts](#ChainGetParentReceipts) - * [ChainGetPath](#ChainGetPath) - * [ChainGetRandomnessFromBeacon](#ChainGetRandomnessFromBeacon) - * [ChainGetRandomnessFromTickets](#ChainGetRandomnessFromTickets) - * [ChainGetTipSet](#ChainGetTipSet) - * [ChainGetTipSetByHeight](#ChainGetTipSetByHeight) - * [ChainHasObj](#ChainHasObj) - * [ChainHead](#ChainHead) - * [ChainNotify](#ChainNotify) - * [ChainReadObj](#ChainReadObj) - * [ChainSetHead](#ChainSetHead) - * [ChainStatObj](#ChainStatObj) - * [ChainTipSetWeight](#ChainTipSetWeight) -* [Client](#Client) - * [ClientCalcCommP](#ClientCalcCommP) - * [ClientDataTransferUpdates](#ClientDataTransferUpdates) - * [ClientDealSize](#ClientDealSize) - * [ClientFindData](#ClientFindData) - * [ClientGenCar](#ClientGenCar) - * [ClientGetDealInfo](#ClientGetDealInfo) - * [ClientGetDealUpdates](#ClientGetDealUpdates) - * [ClientHasLocal](#ClientHasLocal) - * [ClientImport](#ClientImport) - * [ClientListDataTransfers](#ClientListDataTransfers) - * [ClientListDeals](#ClientListDeals) - * [ClientListImports](#ClientListImports) - * [ClientMinerQueryOffer](#ClientMinerQueryOffer) - * [ClientQueryAsk](#ClientQueryAsk) - * [ClientRemoveImport](#ClientRemoveImport) - * [ClientRetrieve](#ClientRetrieve) - * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) - * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) - * [ClientStartDeal](#ClientStartDeal) -* [Gas](#Gas) - * [GasEstimateFeeCap](#GasEstimateFeeCap) - * [GasEstimateGasLimit](#GasEstimateGasLimit) - * [GasEstimateGasPremium](#GasEstimateGasPremium) - * [GasEstimateMessageGas](#GasEstimateMessageGas) -* [I](#I) - * [ID](#ID) -* [Log](#Log) - * [LogList](#LogList) - * [LogSetLevel](#LogSetLevel) -* [Market](#Market) - * [MarketEnsureAvailable](#MarketEnsureAvailable) -* [Miner](#Miner) - * [MinerCreateBlock](#MinerCreateBlock) - * [MinerGetBaseInfo](#MinerGetBaseInfo) -* [Mpool](#Mpool) - * [MpoolClear](#MpoolClear) - * [MpoolGetConfig](#MpoolGetConfig) - * [MpoolGetNonce](#MpoolGetNonce) - * [MpoolPending](#MpoolPending) - * [MpoolPush](#MpoolPush) - * [MpoolPushMessage](#MpoolPushMessage) - * [MpoolSelect](#MpoolSelect) - * [MpoolSetConfig](#MpoolSetConfig) - * [MpoolSub](#MpoolSub) -* [Msig](#Msig) - * [MsigAddApprove](#MsigAddApprove) - * [MsigAddCancel](#MsigAddCancel) - * [MsigAddPropose](#MsigAddPropose) - * [MsigApprove](#MsigApprove) - * [MsigCancel](#MsigCancel) - * [MsigCreate](#MsigCreate) - * [MsigGetAvailableBalance](#MsigGetAvailableBalance) - * [MsigGetVested](#MsigGetVested) - * [MsigPropose](#MsigPropose) - * [MsigSwapApprove](#MsigSwapApprove) - * [MsigSwapCancel](#MsigSwapCancel) - * [MsigSwapPropose](#MsigSwapPropose) -* [Net](#Net) - * [NetAddrsListen](#NetAddrsListen) - * [NetAgentVersion](#NetAgentVersion) - * [NetAutoNatStatus](#NetAutoNatStatus) - * [NetBandwidthStats](#NetBandwidthStats) - * [NetBandwidthStatsByPeer](#NetBandwidthStatsByPeer) - * [NetBandwidthStatsByProtocol](#NetBandwidthStatsByProtocol) - * [NetConnect](#NetConnect) - * [NetConnectedness](#NetConnectedness) - * [NetDisconnect](#NetDisconnect) - * [NetFindPeer](#NetFindPeer) - * [NetPeers](#NetPeers) - * [NetPubsubScores](#NetPubsubScores) -* [Paych](#Paych) - * [PaychAllocateLane](#PaychAllocateLane) - * [PaychAvailableFunds](#PaychAvailableFunds) - * [PaychAvailableFundsByFromTo](#PaychAvailableFundsByFromTo) - * [PaychCollect](#PaychCollect) - * [PaychGet](#PaychGet) - * [PaychGetWaitReady](#PaychGetWaitReady) - * [PaychList](#PaychList) - * [PaychNewPayment](#PaychNewPayment) - * [PaychSettle](#PaychSettle) - * [PaychStatus](#PaychStatus) - * [PaychVoucherAdd](#PaychVoucherAdd) - * [PaychVoucherCheckSpendable](#PaychVoucherCheckSpendable) - * [PaychVoucherCheckValid](#PaychVoucherCheckValid) - * [PaychVoucherCreate](#PaychVoucherCreate) - * [PaychVoucherList](#PaychVoucherList) - * [PaychVoucherSubmit](#PaychVoucherSubmit) -* [State](#State) - * [StateAccountKey](#StateAccountKey) - * [StateAllMinerFaults](#StateAllMinerFaults) - * [StateCall](#StateCall) - * [StateChangedActors](#StateChangedActors) - * [StateCirculatingSupply](#StateCirculatingSupply) - * [StateCompute](#StateCompute) - * [StateDealProviderCollateralBounds](#StateDealProviderCollateralBounds) - * [StateGetActor](#StateGetActor) - * [StateGetReceipt](#StateGetReceipt) - * [StateListActors](#StateListActors) - * [StateListMessages](#StateListMessages) - * [StateListMiners](#StateListMiners) - * [StateLookupID](#StateLookupID) - * [StateMarketBalance](#StateMarketBalance) - * [StateMarketDeals](#StateMarketDeals) - * [StateMarketParticipants](#StateMarketParticipants) - * [StateMarketStorageDeal](#StateMarketStorageDeal) - * [StateMinerActiveSectors](#StateMinerActiveSectors) - * [StateMinerAvailableBalance](#StateMinerAvailableBalance) - * [StateMinerDeadlines](#StateMinerDeadlines) - * [StateMinerFaults](#StateMinerFaults) - * [StateMinerInfo](#StateMinerInfo) - * [StateMinerInitialPledgeCollateral](#StateMinerInitialPledgeCollateral) - * [StateMinerPartitions](#StateMinerPartitions) - * [StateMinerPower](#StateMinerPower) - * [StateMinerPreCommitDepositForPower](#StateMinerPreCommitDepositForPower) - * [StateMinerProvingDeadline](#StateMinerProvingDeadline) - * [StateMinerRecoveries](#StateMinerRecoveries) - * [StateMinerSectorCount](#StateMinerSectorCount) - * [StateMinerSectors](#StateMinerSectors) - * [StateMsgGasCost](#StateMsgGasCost) - * [StateNetworkName](#StateNetworkName) - * [StateReadState](#StateReadState) - * [StateReplay](#StateReplay) - * [StateSearchMsg](#StateSearchMsg) - * [StateSectorExpiration](#StateSectorExpiration) - * [StateSectorGetInfo](#StateSectorGetInfo) - * [StateSectorPartition](#StateSectorPartition) - * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) - * [StateVerifiedClientStatus](#StateVerifiedClientStatus) - * [StateWaitMsg](#StateWaitMsg) -* [Sync](#Sync) - * [SyncCheckBad](#SyncCheckBad) - * [SyncCheckpoint](#SyncCheckpoint) - * [SyncIncomingBlocks](#SyncIncomingBlocks) - * [SyncMarkBad](#SyncMarkBad) - * [SyncState](#SyncState) - * [SyncSubmitBlock](#SyncSubmitBlock) - * [SyncUnmarkBad](#SyncUnmarkBad) -* [Wallet](#Wallet) - * [WalletBalance](#WalletBalance) - * [WalletDefaultAddress](#WalletDefaultAddress) - * [WalletDelete](#WalletDelete) - * [WalletExport](#WalletExport) - * [WalletHas](#WalletHas) - * [WalletImport](#WalletImport) - * [WalletList](#WalletList) - * [WalletNew](#WalletNew) - * [WalletSetDefault](#WalletSetDefault) - * [WalletSign](#WalletSign) - * [WalletSignMessage](#WalletSignMessage) - * [WalletVerify](#WalletVerify) -## - - -### Closing - - -Perms: read - -Inputs: `null` - -Response: `{}` - -### Shutdown - - -Perms: admin - -Inputs: `null` - -Response: `{}` - -### Version - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Version": "string value", - "APIVersion": 3584, - "BlockDelay": 42 -} -``` - -## Auth - - -### AuthNew - - -Perms: admin - -Inputs: -```json -[ - null -] -``` - -Response: `"Ynl0ZSBhcnJheQ=="` - -### AuthVerify - - -Perms: read - -Inputs: -```json -[ - "string value" -] -``` - -Response: `null` - -## Beacon -The Beacon method group contains methods for interacting with the random beacon (DRAND) - - -### BeaconGetEntry -BeaconGetEntry returns the beacon entry for the given filecoin epoch. If -the entry has not yet been produced, the call will block until the entry -becomes available - - -Perms: read - -Inputs: -```json -[ - 10101 -] -``` - -Response: -```json -{ - "Round": 42, - "Data": "Ynl0ZSBhcnJheQ==" -} -``` - -## Chain -The Chain method group contains methods for interacting with the -blockchain, but that do not require any form of state computation. - - -### ChainExport -ChainExport returns a stream of bytes with CAR dump of chain data. -The exported chain data includes the header chain from the given tipset -back to genesis, the entire genesis state, and the most recent 'nroots' -state trees. -If oldmsgskip is set, messages from before the requested roots are also not included. - - -Perms: read - -Inputs: -```json -[ - 10101, - true, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"Ynl0ZSBhcnJheQ=="` - -### ChainGetBlock -ChainGetBlock returns the block specified by the given CID. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Miner": "t01234", - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "ElectionProof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "WinPoStProof": null, - "Parents": null, - "ParentWeight": "0", - "Height": 10101, - "ParentStateRoot": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ParentMessageReceipts": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Messages": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "BLSAggregate": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Timestamp": 42, - "BlockSig": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "ForkSignaling": 42, - "ParentBaseFee": "0" -} -``` - -### ChainGetBlockMessages -ChainGetBlockMessages returns messages stored in the specified block. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "BlsMessages": null, - "SecpkMessages": null, - "Cids": null -} -``` - -### ChainGetGenesis -ChainGetGenesis returns the genesis tipset. - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Cids": null, - "Blocks": null, - "Height": 0 -} -``` - -### ChainGetMessage -ChainGetMessage reads a message referenced by the specified CID from the -chain blockstore. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" -} -``` - -### ChainGetNode -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "string value" -] -``` - -Response: -```json -{ - "Cid": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Obj": {} -} -``` - -### ChainGetParentMessages -ChainGetParentMessages returns messages stored in parent tipset of the -specified block. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `null` - -### ChainGetParentReceipts -ChainGetParentReceipts returns receipts for messages in parent tipset of -the specified block. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `null` - -### ChainGetPath -ChainGetPath returns a set of revert/apply operations needed to get from -one tipset to another, for example: -``` - to - ^ -from tAA - ^ ^ -tBA tAB - ^---*--^ - ^ - tRR -``` -Would return `[revert(tBA), apply(tAB), apply(tAA)]` - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### ChainGetRandomnessFromBeacon -ChainGetRandomnessFromBeacon is used to sample the beacon for randomness. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - 2, - 10101, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: `null` - -### ChainGetRandomnessFromTickets -ChainGetRandomnessFromTickets is used to sample the chain for randomness. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - 2, - 10101, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: `null` - -### ChainGetTipSet -ChainGetTipSet returns the tipset specified by the given TipSetKey. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Cids": null, - "Blocks": null, - "Height": 0 -} -``` - -### ChainGetTipSetByHeight -ChainGetTipSetByHeight looks back for a tipset at the specified epoch. -If there are no blocks at the specified epoch, a tipset at an earlier epoch -will be returned. - - -Perms: read - -Inputs: -```json -[ - 10101, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Cids": null, - "Blocks": null, - "Height": 0 -} -``` - -### ChainHasObj -ChainHasObj checks if a given CID exists in the chain blockstore. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `true` - -### ChainHead -ChainHead returns the current head of the chain. - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Cids": null, - "Blocks": null, - "Height": 0 -} -``` - -### ChainNotify -ChainNotify returns channel with chain head updates. -First message is guaranteed to be of len == 1, and type == 'current'. - - -Perms: read - -Inputs: `null` - -Response: `null` - -### ChainReadObj -ChainReadObj reads ipld nodes referenced by the specified CID from chain -blockstore and returns raw bytes. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `"Ynl0ZSBhcnJheQ=="` - -### ChainSetHead -ChainSetHead forcefully sets current chain head. Use with caution. - - -Perms: admin - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `{}` - -### ChainStatObj -ChainStatObj returns statistics about the graph referenced by 'obj'. -If 'base' is also specified, then the returned stat will be a diff -between the two objects. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Size": 42, - "Links": 42 -} -``` - -### ChainTipSetWeight -ChainTipSetWeight computes weight for the specified tipset. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -## Client -The Client methods all have to do with interacting with the storage and -retrieval markets as a client - - -### ClientCalcCommP -ClientCalcCommP calculates the CommP for a specified file - - -Perms: read - -Inputs: -```json -[ - "string value" -] -``` - -Response: -```json -{ - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Size": 1024 -} -``` - -### ClientDataTransferUpdates -There are not yet any comments for this method. - -Perms: write - -Inputs: `null` - -Response: -```json -{ - "TransferID": 3, - "Status": 1, - "BaseCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "IsInitiator": true, - "IsSender": true, - "Voucher": "string value", - "Message": "string value", - "OtherPeer": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "Transferred": 42 -} -``` - -### ClientDealSize -ClientDealSize calculates real deal data size - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "PayloadSize": 9, - "PieceSize": 1032 -} -``` - -### ClientFindData -ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer). - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - null -] -``` - -Response: `null` - -### ClientGenCar -ClientGenCar generates a CAR file for the specified file. - - -Perms: write - -Inputs: -```json -[ - { - "Path": "string value", - "IsCAR": true - }, - "string value" -] -``` - -Response: `{}` - -### ClientGetDealInfo -ClientGetDealInfo returns the latest information about a given deal. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "ProposalCid": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "State": 42, - "Message": "string value", - "Provider": "t01234", - "DataRef": { - "TransferType": "string value", - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceCid": null, - "PieceSize": 1024 - }, - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Size": 42, - "PricePerEpoch": "0", - "Duration": 42, - "DealID": 5432, - "CreationTime": "0001-01-01T00:00:00Z" -} -``` - -### ClientGetDealUpdates -ClientGetDealUpdates returns the status of updated deals - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "ProposalCid": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "State": 42, - "Message": "string value", - "Provider": "t01234", - "DataRef": { - "TransferType": "string value", - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceCid": null, - "PieceSize": 1024 - }, - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Size": 42, - "PricePerEpoch": "0", - "Duration": 42, - "DealID": 5432, - "CreationTime": "0001-01-01T00:00:00Z" -} -``` - -### ClientHasLocal -ClientHasLocal indicates whether a certain CID is locally stored. - - -Perms: write - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `true` - -### ClientImport -ClientImport imports file under the specified path into filestore. - - -Perms: admin - -Inputs: -```json -[ - { - "Path": "string value", - "IsCAR": true - } -] -``` - -Response: -```json -{ - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ImportID": 50 -} -``` - -### ClientListDataTransfers -ClientListTransfers returns the status of all ongoing transfers of data - - -Perms: write - -Inputs: `null` - -Response: `null` - -### ClientListDeals -ClientListDeals returns information about the deals made by the local client. - - -Perms: write - -Inputs: `null` - -Response: `null` - -### ClientListImports -ClientListImports lists imported files and their root CIDs - - -Perms: write - -Inputs: `null` - -Response: `null` - -### ClientMinerQueryOffer -ClientMinerQueryOffer returns a QueryOffer for the specific miner and file. - - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - null -] -``` - -Response: -```json -{ - "Err": "string value", - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Piece": null, - "Size": 42, - "MinPrice": "0", - "UnsealPrice": "0", - "PaymentInterval": 42, - "PaymentIntervalIncrease": 42, - "Miner": "t01234", - "MinerPeer": { - "Address": "t01234", - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "PieceCID": null - } -} -``` - -### ClientQueryAsk -ClientQueryAsk returns a signed StorageAsk from the specified miner. - - -Perms: read - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "t01234" -] -``` - -Response: -```json -{ - "Ask": { - "Price": "0", - "VerifiedPrice": "0", - "MinPieceSize": 1032, - "MaxPieceSize": 1032, - "Miner": "t01234", - "Timestamp": 10101, - "Expiry": 10101, - "SeqNo": 42 - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } -} -``` - -### ClientRemoveImport -ClientRemoveImport removes file import - - -Perms: admin - -Inputs: -```json -[ - 50 -] -``` - -Response: `{}` - -### ClientRetrieve -ClientRetrieve initiates the retrieval of a file, as specified in the order. - - -Perms: admin - -Inputs: -```json -[ - { - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Piece": null, - "Size": 42, - "Total": "0", - "UnsealPrice": "0", - "PaymentInterval": 42, - "PaymentIntervalIncrease": 42, - "Client": "t01234", - "Miner": "t01234", - "MinerPeer": { - "Address": "t01234", - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "PieceCID": null - } - }, - { - "Path": "string value", - "IsCAR": true - } -] -``` - -Response: `{}` - -### ClientRetrieveTryRestartInsufficientFunds -ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel -which are stuck due to insufficient funds - - -Perms: write - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `{}` - -### ClientRetrieveWithEvents -ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel -of status updates. - - -Perms: admin - -Inputs: -```json -[ - { - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Piece": null, - "Size": 42, - "Total": "0", - "UnsealPrice": "0", - "PaymentInterval": 42, - "PaymentIntervalIncrease": 42, - "Client": "t01234", - "Miner": "t01234", - "MinerPeer": { - "Address": "t01234", - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "PieceCID": null - } - }, - { - "Path": "string value", - "IsCAR": true - } -] -``` - -Response: -```json -{ - "Event": 5, - "Status": 0, - "BytesReceived": 42, - "FundsSpent": "0", - "Err": "string value" -} -``` - -### ClientStartDeal -ClientStartDeal proposes a deal with a miner. - - -Perms: admin - -Inputs: -```json -[ - { - "Data": { - "TransferType": "string value", - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceCid": null, - "PieceSize": 1024 - }, - "Wallet": "t01234", - "Miner": "t01234", - "EpochPrice": "0", - "MinBlocksDuration": 42, - "ProviderCollateral": "0", - "DealStartEpoch": 10101, - "FastRetrieval": true, - "VerifiedDeal": true - } -] -``` - -Response: `null` - -## Gas - - -### GasEstimateFeeCap -GasEstimateFeeCap estimates gas fee cap - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### GasEstimateGasLimit -GasEstimateGasLimit estimates gas used by the message and returns it. -It fails if message fails to execute. - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `9` - -### GasEstimateGasPremium -GasEstimateGasPremium estimates what gas price should be used for a -message to have high likelihood of inclusion in `nblocksincl` epochs. - - -Perms: read - -Inputs: -```json -[ - 42, - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### GasEstimateMessageGas -GasEstimateMessageGas estimates gas values for unset message gas fields - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - { - "MaxFee": "0" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" -} -``` - -## I - - -### ID - - -Perms: read - -Inputs: `null` - -Response: `"12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"` - -## Log - - -### LogList - - -Perms: write - -Inputs: `null` - -Response: `null` - -### LogSetLevel - - -Perms: write - -Inputs: -```json -[ - "string value", - "string value" -] -``` - -Response: `{}` - -## Market - - -### MarketEnsureAvailable -MarketFreeBalance - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "0" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -## Miner - - -### MinerCreateBlock -There are not yet any comments for this method. - -Perms: write - -Inputs: -```json -[ - { - "Miner": "t01234", - "Parents": [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "Eproof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconValues": null, - "Messages": null, - "Epoch": 10101, - "Timestamp": 42, - "WinningPoStProof": null - } -] -``` - -Response: -```json -{ - "Header": { - "Miner": "t01234", - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "ElectionProof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "WinPoStProof": null, - "Parents": null, - "ParentWeight": "0", - "Height": 10101, - "ParentStateRoot": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ParentMessageReceipts": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Messages": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "BLSAggregate": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Timestamp": 42, - "BlockSig": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "ForkSignaling": 42, - "ParentBaseFee": "0" - }, - "BlsMessages": null, - "SecpkMessages": null -} -``` - -### MinerGetBaseInfo -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "t01234", - 10101, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "MinerPower": "0", - "NetworkPower": "0", - "Sectors": null, - "WorkerKey": "t01234", - "SectorSize": 34359738368, - "PrevBeaconEntry": { - "Round": 42, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "HasMinPower": true -} -``` - -## Mpool -The Mpool methods are for interacting with the message pool. The message pool -manages all incoming and outgoing 'messages' going over the network. - - -### MpoolClear -MpoolClear clears pending messages from the mpool - - -Perms: write - -Inputs: -```json -[ - true -] -``` - -Response: `{}` - -### MpoolGetConfig -MpoolGetConfig returns (a copy of) the current mpool config - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "PriorityAddrs": null, - "SizeLimitHigh": 123, - "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, - "PruneCooldown": 60000000000, - "GasLimitOverestimation": 12.3 -} -``` - -### MpoolGetNonce -MpoolGetNonce gets next nonce for the specified sender. -Note that this method may not be atomic. Use MpoolPushMessage instead. - - -Perms: read - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `42` - -### MpoolPending -MpoolPending returns pending mempool messages. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### MpoolPush -MpoolPush pushes a signed message to mempool. - - -Perms: write - -Inputs: -```json -[ - { - "Message": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - } -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MpoolPushMessage -MpoolPushMessage atomically assigns a nonce, signs, and pushes a message -to mempool. -maxFee is only used when GasFeeCap/GasPremium fields aren't specified - -When maxFee is set to 0, MpoolPushMessage will guess appropriate fee -based on current chain conditions - - -Perms: sign - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - { - "MaxFee": "0" - } -] -``` - -Response: -```json -{ - "Message": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } -} -``` - -### MpoolSelect -MpoolSelect returns a list of pending messages for inclusion in the next block - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - 12.3 -] -``` - -Response: `null` - -### MpoolSetConfig -MpoolSetConfig sets the mpool config to (a copy of) the supplied config - - -Perms: write - -Inputs: -```json -[ - { - "PriorityAddrs": null, - "SizeLimitHigh": 123, - "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, - "PruneCooldown": 60000000000, - "GasLimitOverestimation": 12.3 - } -] -``` - -Response: `{}` - -### MpoolSub -There are not yet any comments for this method. - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Type": 0, - "Message": { - "Message": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - } -} -``` - -## Msig -The Msig methods are used to interact with multisig wallets on the -filecoin network - - -### MsigAddApprove -MsigAddApprove approves a previously proposed AddSigner message -It takes the following params: , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - 42, - "t01234", - "t01234", - true -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigAddCancel -MsigAddCancel cancels a previously proposed AddSigner message -It takes the following params: , , , -, - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - 42, - "t01234", - true -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigAddPropose -MsigAddPropose proposes adding a signer in the multisig -It takes the following params: , , -, - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "t01234", - true -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigApprove -MsigApprove approves a previously-proposed multisig message -It takes the following params: , , , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - 42, - "t01234", - "t01234", - "0", - "t01234", - 42, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigCancel -MsigCancel cancels a previously-proposed multisig message -It takes the following params: , , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - 42, - "t01234", - "0", - "t01234", - 42, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigCreate -MsigCreate creates a multisig wallet -It takes the following params: , , -, , - - -Perms: sign - -Inputs: -```json -[ - 42, - null, - 10101, - "0", - "t01234", - "0" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigGetAvailableBalance -MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### MsigGetVested -MsigGetVested returns the amount of FIL that vested in a multisig in a certain period. -It takes the following params: , , - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### MsigPropose -MsigPropose proposes a multisig message -It takes the following params: , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "0", - "t01234", - 42, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigSwapApprove -MsigSwapApprove approves a previously proposed SwapSigner -It takes the following params: , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - 42, - "t01234", - "t01234", - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigSwapCancel -MsigSwapCancel cancels a previously proposed SwapSigner message -It takes the following params: , , , -, - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - 42, - "t01234", - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigSwapPropose -MsigSwapPropose proposes swapping 2 signers in the multisig -It takes the following params: , , -, - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "t01234", - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -## Net - - -### NetAddrsListen - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Addrs": null, - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -} -``` - -### NetAgentVersion - - -Perms: read - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -] -``` - -Response: `"string value"` - -### NetAutoNatStatus - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Reachability": 1, - "PublicAddr": "string value" -} -``` - -### NetBandwidthStats - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "TotalIn": 9, - "TotalOut": 9, - "RateIn": 12.3, - "RateOut": 12.3 -} -``` - -### NetBandwidthStatsByPeer - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "12D3KooWSXmXLJmBR1M7i9RW9GQPNUhZSzXKzxDHWtAgNuJAbyEJ": { - "TotalIn": 174000, - "TotalOut": 12500, - "RateIn": 100, - "RateOut": 50 - } -} -``` - -### NetBandwidthStatsByProtocol - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "/fil/hello/1.0.0": { - "TotalIn": 174000, - "TotalOut": 12500, - "RateIn": 100, - "RateOut": 50 - } -} -``` - -### NetConnect - - -Perms: write - -Inputs: -```json -[ - { - "Addrs": null, - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" - } -] -``` - -Response: `{}` - -### NetConnectedness - - -Perms: read - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -] -``` - -Response: `1` - -### NetDisconnect - - -Perms: write - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -] -``` - -Response: `{}` - -### NetFindPeer - - -Perms: read - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -] -``` - -Response: -```json -{ - "Addrs": null, - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -} -``` - -### NetPeers - - -Perms: read - -Inputs: `null` - -Response: `null` - -### NetPubsubScores - - -Perms: read - -Inputs: `null` - -Response: `null` - -## Paych -The Paych methods are for interacting with and managing payment channels - - -### PaychAllocateLane -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `42` - -### PaychAvailableFunds -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "Channel": "\u003cempty\u003e", - "From": "t01234", - "To": "t01234", - "ConfirmedAmt": "0", - "PendingAmt": "0", - "PendingWaitSentinel": null, - "QueuedAmt": "0", - "VoucherReedeemedAmt": "0" -} -``` - -### PaychAvailableFundsByFromTo -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234" -] -``` - -Response: -```json -{ - "Channel": "\u003cempty\u003e", - "From": "t01234", - "To": "t01234", - "ConfirmedAmt": "0", - "PendingAmt": "0", - "PendingWaitSentinel": null, - "QueuedAmt": "0", - "VoucherReedeemedAmt": "0" -} -``` - -### PaychCollect -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### PaychGet -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "0" -] -``` - -Response: -```json -{ - "Channel": "t01234", - "WaitSentinel": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -} -``` - -### PaychGetWaitReady -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `"t01234"` - -### PaychList -There are not yet any comments for this method. - -Perms: read - -Inputs: `null` - -Response: `null` - -### PaychNewPayment -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - null -] -``` - -Response: -```json -{ - "Channel": "t01234", - "WaitSentinel": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Vouchers": null -} -``` - -### PaychSettle -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### PaychStatus -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "ControlAddr": "t01234", - "Direction": 1 -} -``` - -### PaychVoucherAdd -There are not yet any comments for this method. - -Perms: write - -Inputs: -```json -[ - "t01234", - { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - }, - "Ynl0ZSBhcnJheQ==", - "0" -] -``` - -Response: `"0"` - -### PaychVoucherCheckSpendable -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - }, - "Ynl0ZSBhcnJheQ==", - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: `true` - -### PaychVoucherCheckValid -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - } -] -``` - -Response: `{}` - -### PaychVoucherCreate -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - "0", - 42 -] -``` - -Response: -```json -{ - "Voucher": { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - }, - "Shortfall": "0" -} -``` - -### PaychVoucherList -There are not yet any comments for this method. - -Perms: write - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `null` - -### PaychVoucherSubmit -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - }, - "Ynl0ZSBhcnJheQ==", - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -## State -The State methods are used to query, inspect, and interact with chain state. -All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. -A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. - - -### StateAccountKey -StateAccountKey returns the public key address of the given ID address - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"t01234"` - -### StateAllMinerFaults -StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset - - -Perms: read - -Inputs: -```json -[ - 10101, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateCall -StateCall runs the given message and returns its result without any persisted changes. - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Msg": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "ExecutionTrace": { - "Msg": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "Error": "string value", - "Duration": 60000000000, - "GasCharges": null, - "Subcalls": null - }, - "Error": "string value", - "Duration": 60000000000 -} -``` - -### StateChangedActors -StateChangedActors returns all the actors whose states change between the two given state CIDs -TODO: Should this take tipset keys instead? - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "t01236": { - "Code": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Head": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Nonce": 42, - "Balance": "0" - } -} -``` - -### StateCirculatingSupply -StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "FilVested": "0", - "FilMined": "0", - "FilBurnt": "0", - "FilLocked": "0", - "FilCirculating": "0" -} -``` - -### StateCompute -StateCompute is a flexible command that applies the given messages on the given tipset. -The messages are run as though the VM were at the provided height. - - -Perms: read - -Inputs: -```json -[ - 10101, - null, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Trace": null -} -``` - -### StateDealProviderCollateralBounds -StateDealProviderCollateralBounds returns the min and max collateral a storage provider -can issue. It takes the deal size and verified status as parameters. - - -Perms: read - -Inputs: -```json -[ - 1032, - true, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Min": "0", - "Max": "0" -} -``` - -### StateGetActor -StateGetActor returns the indicated actor's nonce and balance. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Code": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Head": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Nonce": 42, - "Balance": "0" -} -``` - -### StateGetReceipt -StateGetReceipt returns the message receipt for the given message - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 -} -``` - -### StateListActors -StateListActors returns the addresses of every actor in the state - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateListMessages -StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height. - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - 10101 -] -``` - -Response: `null` - -### StateListMiners -StateListMiners returns the addresses of every miner that has claimed power in the Power Actor - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateLookupID -StateLookupID retrieves the ID address of the given address - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"t01234"` - -### StateMarketBalance -StateMarketBalance looks up the Escrow and Locked balances of the given address in the Storage Market - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Escrow": "0", - "Locked": "0" -} -``` - -### StateMarketDeals -StateMarketDeals returns information about every deal in the Storage Market - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "t026363": { - "Proposal": { - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceSize": 1032, - "VerifiedDeal": true, - "Client": "t01234", - "Provider": "t01234", - "Label": "string value", - "StartEpoch": 10101, - "EndEpoch": 10101, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "State": { - "SectorStartEpoch": 10101, - "LastUpdatedEpoch": 10101, - "SlashEpoch": 10101 - } - } -} -``` - -### StateMarketParticipants -StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "t026363": { - "Escrow": "0", - "Locked": "0" - } -} -``` - -### StateMarketStorageDeal -StateMarketStorageDeal returns information about the indicated deal - - -Perms: read - -Inputs: -```json -[ - 5432, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Proposal": { - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceSize": 1032, - "VerifiedDeal": true, - "Client": "t01234", - "Provider": "t01234", - "Label": "string value", - "StartEpoch": 10101, - "EndEpoch": 10101, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "State": { - "SectorStartEpoch": 10101, - "LastUpdatedEpoch": 10101, - "SlashEpoch": 10101 - } -} -``` - -### StateMinerActiveSectors -StateMinerActiveSectors returns info about sectors that a given miner is actively proving. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateMinerAvailableBalance -StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### StateMinerDeadlines -StateMinerDeadlines returns all the proving deadlines for the given miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateMinerFaults -StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -[ - 5, - 1 -] -``` - -### StateMinerInfo -StateMinerInfo returns info about the indicated miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Owner": "t01234", - "Worker": "t01234", - "NewWorker": "t01234", - "ControlAddresses": null, - "WorkerChangeEpoch": 10101, - "PeerId": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "Multiaddrs": null, - "SealProofType": 3, - "SectorSize": 34359738368, - "WindowPoStPartitionSectors": 42 -} -``` - -### StateMinerInitialPledgeCollateral -StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector - - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "SealProof": 3, - "SectorNumber": 9, - "SealedCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "SealRandEpoch": 10101, - "DealIDs": null, - "Expiration": 10101, - "ReplaceCapacity": true, - "ReplaceSectorDeadline": 42, - "ReplaceSectorPartition": 42, - "ReplaceSectorNumber": 9 - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### StateMinerPartitions -StateMinerPartitions loads miner partitions for the specified miner/deadline - - -Perms: read - -Inputs: -```json -[ - "t01234", - 42, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateMinerPower -StateMinerPower returns the power of the indicated miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "MinerPower": { - "RawBytePower": "0", - "QualityAdjPower": "0" - }, - "TotalPower": { - "RawBytePower": "0", - "QualityAdjPower": "0" - } -} -``` - -### StateMinerPreCommitDepositForPower -StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector - - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "SealProof": 3, - "SectorNumber": 9, - "SealedCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "SealRandEpoch": 10101, - "DealIDs": null, - "Expiration": 10101, - "ReplaceCapacity": true, - "ReplaceSectorDeadline": 42, - "ReplaceSectorPartition": 42, - "ReplaceSectorNumber": 9 - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### StateMinerProvingDeadline -StateMinerProvingDeadline calculates the deadline at some epoch for a proving period -and returns the deadline-related calculations. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "CurrentEpoch": 10101, - "PeriodStart": 10101, - "Index": 42, - "Open": 10101, - "Close": 10101, - "Challenge": 10101, - "FaultCutoff": 10101, - "WPoStPeriodDeadlines": 42, - "WPoStProvingPeriod": 10101, - "WPoStChallengeWindow": 10101, - "WPoStChallengeLookback": 10101, - "FaultDeclarationCutoff": 10101 -} -``` - -### StateMinerRecoveries -StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -[ - 5, - 1 -] -``` - -### StateMinerSectorCount -StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Sectors": 42, - "Active": 42 -} -``` - -### StateMinerSectors -StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included. -If the filterOut boolean is set to true, any sectors in the filter are excluded. -If false, only those sectors in the filter are included. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - 0 - ], - true, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateMsgGasCost -StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "GasUsed": "0", - "BaseFeeBurn": "0", - "OverEstimationBurn": "0", - "MinerPenalty": "0", - "MinerTip": "0", - "Refund": "0", - "TotalCost": "0" -} -``` - -### StateNetworkName -StateNetworkName returns the name of the network the node is synced to - - -Perms: read - -Inputs: `null` - -Response: `"lotus"` - -### StateReadState -StateReadState returns the indicated actor's state. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Balance": "0", - "State": {} -} -``` - -### StateReplay -StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Msg": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "ExecutionTrace": { - "Msg": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "Error": "string value", - "Duration": 60000000000, - "GasCharges": null, - "Subcalls": null - }, - "Error": "string value", - "Duration": 60000000000 -} -``` - -### StateSearchMsg -StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Receipt": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "ReturnDec": {}, - "TipSet": [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - "Height": 10101 -} -``` - -### StateSectorExpiration -StateSectorExpiration returns epoch at which given sector will expire - - -Perms: read - -Inputs: -```json -[ - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "OnTime": 10101, - "Early": 10101 -} -``` - -### StateSectorGetInfo -StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found -NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate -expiration epoch - - -Perms: read - -Inputs: -```json -[ - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "SectorNumber": 9, - "SealProof": 3, - "SealedCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "DealIDs": null, - "Activation": 10101, - "Expiration": 10101, - "DealWeight": "0", - "VerifiedDealWeight": "0", - "InitialPledge": "0", - "ExpectedDayReward": "0", - "ExpectedStoragePledge": "0" -} -``` - -### StateSectorPartition -StateSectorPartition finds deadline/partition with the specified sector - - -Perms: read - -Inputs: -```json -[ - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Deadline": 42, - "Partition": 42 -} -``` - -### StateSectorPreCommitInfo -StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector - - -Perms: read - -Inputs: -```json -[ - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Info": { - "SealProof": 3, - "SectorNumber": 9, - "SealedCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "SealRandEpoch": 10101, - "DealIDs": null, - "Expiration": 10101, - "ReplaceCapacity": true, - "ReplaceSectorDeadline": 42, - "ReplaceSectorPartition": 42, - "ReplaceSectorNumber": 9 - }, - "PreCommitDeposit": "0", - "PreCommitEpoch": 10101, - "DealWeight": "0", - "VerifiedDealWeight": "0" -} -``` - -### StateVerifiedClientStatus -StateVerifiedClientStatus returns the data cap for the given address. -Returns nil if there is no entry in the data cap table for the -address. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### StateWaitMsg -StateWaitMsg looks back in the chain for a message. If not found, it blocks until the -message arrives on chain, and gets to the indicated confidence depth. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - 42 -] -``` - -Response: -```json -{ - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Receipt": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "ReturnDec": {}, - "TipSet": [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - "Height": 10101 -} -``` - -## Sync -The Sync method group contains methods for interacting with and -observing the lotus sync service. - - -### SyncCheckBad -SyncCheckBad checks if a block was marked as bad, and if it was, returns -the reason. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `"string value"` - -### SyncCheckpoint -SyncCheckpoint marks a blocks as checkpointed, meaning that it won't ever fork away from it. - - -Perms: admin - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `{}` - -### SyncIncomingBlocks -SyncIncomingBlocks returns a channel streaming incoming, potentially not -yet synced block headers. - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Miner": "t01234", - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "ElectionProof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "WinPoStProof": null, - "Parents": null, - "ParentWeight": "0", - "Height": 10101, - "ParentStateRoot": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ParentMessageReceipts": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Messages": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "BLSAggregate": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Timestamp": 42, - "BlockSig": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "ForkSignaling": 42, - "ParentBaseFee": "0" -} -``` - -### SyncMarkBad -SyncMarkBad marks a blocks as bad, meaning that it won't ever by synced. -Use with extreme caution. - - -Perms: admin - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `{}` - -### SyncState -SyncState returns the current status of the lotus sync system. - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "ActiveSyncs": null -} -``` - -### SyncSubmitBlock -SyncSubmitBlock can be used to submit a newly created block to the. -network through this node - - -Perms: write - -Inputs: -```json -[ - { - "Header": { - "Miner": "t01234", - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "ElectionProof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "WinPoStProof": null, - "Parents": null, - "ParentWeight": "0", - "Height": 10101, - "ParentStateRoot": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ParentMessageReceipts": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Messages": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "BLSAggregate": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Timestamp": 42, - "BlockSig": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "ForkSignaling": 42, - "ParentBaseFee": "0" - }, - "BlsMessages": null, - "SecpkMessages": null - } -] -``` - -Response: `{}` - -### SyncUnmarkBad -SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. - - -Perms: admin - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `{}` - -## Wallet - - -### WalletBalance -WalletBalance returns the balance of the given address at the current head of the chain. - - -Perms: read - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `"0"` - -### WalletDefaultAddress -WalletDefaultAddress returns the address marked as default in the wallet. - - -Perms: write - -Inputs: `null` - -Response: `"t01234"` - -### WalletDelete -WalletDelete deletes an address from the wallet. - - -Perms: write - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `{}` - -### WalletExport -WalletExport returns the private key of an address in the wallet. - - -Perms: admin - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "Type": "string value", - "PrivateKey": "Ynl0ZSBhcnJheQ==" -} -``` - -### WalletHas -WalletHas indicates whether the given address is in the wallet. - - -Perms: write - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `true` - -### WalletImport -WalletImport receives a KeyInfo, which includes a private key, and imports it into the wallet. - - -Perms: admin - -Inputs: -```json -[ - { - "Type": "string value", - "PrivateKey": "Ynl0ZSBhcnJheQ==" - } -] -``` - -Response: `"t01234"` - -### WalletList -WalletList lists all the addresses in the wallet. - - -Perms: write - -Inputs: `null` - -Response: `null` - -### WalletNew -WalletNew creates a new address in the wallet with the given sigType. - - -Perms: write - -Inputs: -```json -[ - 2 -] -``` - -Response: `"t01234"` - -### WalletSetDefault -WalletSetDefault marks the given address as as the default one. - - -Perms: admin - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `{}` - -### WalletSign -WalletSign signs the given bytes using the given address. - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" -} -``` - -### WalletSignMessage -WalletSignMessage signs the given message using the given address. - - -Perms: sign - -Inputs: -```json -[ - "t01234", - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - } -] -``` - -Response: -```json -{ - "Message": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } -} -``` - -### WalletVerify -WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid. -The address does not have to be in the wallet. - - -Perms: read - -Inputs: -```json -[ - "t01234", - "Ynl0ZSBhcnJheQ==", - { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } -] -``` - -Response: `true` - diff --git a/documentation/en/building/api-troubleshooting.md b/documentation/en/building/api-troubleshooting.md deleted file mode 100644 index 0cb3a6800..000000000 --- a/documentation/en/building/api-troubleshooting.md +++ /dev/null @@ -1,36 +0,0 @@ -# API Troubleshooting - -## Types: params - -`params` must be an array. If there are no `params` you should still pass an empty array. - -## Types: TipSet - -For methods such as `Filecoin.StateMinerPower`, where the method accepts the argument of the type `TipSet`, you can pass `null` to use the current chain head. - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - --data '{ "jsonrpc": "2.0", "method": "Filecoin.StateMinerPower", "params": ["t0101", null], "id": 3 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -## Types: Sending a CID - -If you do not serialize the CID as a [JSON IPLD link](https://did-ipid.github.io/ipid-did-method/#txref), you will receive an error. Here is an example of a broken CURL request: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - --data '{ "jsonrpc": "2.0", "method":"Filecoin.ClientGetDealInfo", "params": ["bafyreiaxl446wlnu6t6dpq4ivrjf4gda4gvsoi4rr6mpxau7z25xvk5pl4"], "id": 0 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -To fix it, change the `params` property to: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - --data '{ "jsonrpc": "2.0", "method":"Filecoin.ClientGetDealInfo", "params": [{"/": "bafyreiaxl446wlnu6t6dpq4ivrjf4gda4gvsoi4rr6mpxau7z25xvk5pl4"}], "id": 0 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` diff --git a/documentation/en/building/api.md b/documentation/en/building/api.md deleted file mode 100644 index 3a2c2902b..000000000 --- a/documentation/en/building/api.md +++ /dev/null @@ -1,38 +0,0 @@ -# API endpoints and methods - -The API can be accessed on: - -- `http://[api:port]/rpc/v0` - HTTP RPC-API endpoint -- `ws://[api:port]/rpc/v0` - Websocket RPC-API endpoint -- `PUT http://[api:port]/rest/v0/import` - REST endpoint for file import (multipart upload). It requires write permissions. - -The RPC methods can be found in the [Reference](en+api-methods) and directly in the source code: - -- [Both Lotus node + miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go) -- [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) -- [Lotus miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go) - - -## JSON-RPC client - -Lotus uses its own Go library implementation of [JSON-RPC](https://github.com/filecoin-project/go-jsonrpc). - -## cURL example - -To demonstrate making an API request, we will take the method `ChainHead` from [api/api_full.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). - -```go -ChainHead(context.Context) (*types.TipSet, error) -``` - -And create a CURL command. In this command, `ChainHead` is included as `{ "method": "Filecoin.ChainHead" }`: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $(cat ~/.lotusminer/token)" \ - --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -(See [this section](en+remote-api) to learn how to generate authorization tokens). diff --git a/documentation/en/building/building.md b/documentation/en/building/building.md deleted file mode 100644 index 5194f8314..000000000 --- a/documentation/en/building/building.md +++ /dev/null @@ -1,5 +0,0 @@ -# Building with Lotus - -Lotus applications provide HTTP (JSON-RPC) APIs that allow developers to control Lotus programatically. - -This section dives into how to setup and use these APIs, additionally providing information on advanced Lotus features and workflows, like Payment Channels or how to setup a fully local Lotus development network. diff --git a/documentation/en/building/jaeger-tracing.md b/documentation/en/building/jaeger-tracing.md deleted file mode 100644 index bbe4d3052..000000000 --- a/documentation/en/building/jaeger-tracing.md +++ /dev/null @@ -1,26 +0,0 @@ -# Jaeger Tracing - -Lotus has tracing built into many of its internals. To view the traces, first download [Jaeger](https://www.jaegertracing.io/download/) (Choose the 'all-in-one' binary). Then run it somewhere, start up the lotus daemon, and open up localhost:16686 in your browser. - -## Open Census - -Lotus uses [OpenCensus](https://opencensus.io/) for tracing application flow. This generates spans through the execution of annotated code paths. - -Currently it is set up to use Jaeger, though other tracing backends should be fairly easy to swap in. - -## Running Locally - -To easily run and view tracing locally, first, install jaeger. The easiest way to do this is to [download the binaries](https://www.jaegertracing.io/download/) and then run the `jaeger-all-in-one` binary. This will start up jaeger, listen for spans on `localhost:6831`, and expose a web UI for viewing traces on `http://localhost:16686/`. - -Now, to start sending traces from Lotus to Jaeger, set the environment variable `LOTUS_JAEGER` to `localhost:6831`, and start the `lotus daemon`. - -Now, to view any generated traces, open up `http://localhost:16686/` in your browser. - -## Adding Spans - -To annotate a new codepath with spans, add the following lines to the top of the function you wish to trace: - -```go -ctx, span := trace.StartSpan(ctx, "put function name here") -defer span.End() -``` diff --git a/documentation/en/building/local-devnet.md b/documentation/en/building/local-devnet.md deleted file mode 100644 index 3382b6471..000000000 --- a/documentation/en/building/local-devnet.md +++ /dev/null @@ -1,54 +0,0 @@ -# Setup Local Devnet - -Build the Lotus Binaries in debug mode, This enables the use of 2048 byte sectors. - -```sh -make 2k -``` - -Set the `LOTUS_SKIP_GENESIS_CHECK` environment variable to `_yes_`. This tells your -Lotus node that it's okay if the genesis being used doesn't match any baked-in -genesis. - -```sh -export LOTUS_SKIP_GENESIS_CHECK=_yes_ -``` - -Download the 2048 byte parameters: -```sh -./lotus fetch-params 2048 -``` - -Pre-seal some sectors: - -```sh -./lotus-seed pre-seal --sector-size 2KiB --num-sectors 2 -``` - -Create the genesis block and start up the first node: - -```sh -./lotus-seed genesis new localnet.json -./lotus-seed genesis add-miner localnet.json ~/.genesis-sectors/pre-seal-t01000.json -./lotus daemon --lotus-make-genesis=devgen.car --genesis-template=localnet.json --bootstrap=false -``` - -Then, in another console, import the genesis miner key: - -```sh -./lotus wallet import --as-default ~/.genesis-sectors/pre-seal-t01000.key -``` - -Set up the genesis miner: - -```sh -./lotus-miner init --genesis-miner --actor=t01000 --sector-size=2KiB --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json --nosync -``` - -Now, finally, start up the miner: - -```sh -./lotus-miner run --nosync -``` - -If all went well, you will have your own local Lotus Devnet running. diff --git a/documentation/en/building/payment-channels.md b/documentation/en/building/payment-channels.md deleted file mode 100644 index afddcdc40..000000000 --- a/documentation/en/building/payment-channels.md +++ /dev/null @@ -1,111 +0,0 @@ -# Payment Channels - -Payment channels are used to transfer funds between two actors. - -For example in lotus a payment channel is created when a client wants to fetch data from a provider. -The client sends vouchers for the payment channel, and the provider sends data in response. - -The payment channel is created on-chain with an initial amount. -Vouchers allow the client and the provider to exchange funds incrementally off-chain. -The provider can submit vouchers to chain at any stage. -Either party to the payment channel can settle the payment channel on chain. -After a settlement period (currently 12 hours) either party to the payment channel can call collect on chain. -Collect sends the value of submitted vouchers to the channel recipient (the provider), and refunds the remaining channel balance to the channel creator (the client). - -Vouchers have a lane, a nonce and a value, where vouchers with a higher nonce supersede vouchers with a lower nonce in the same lane. -Each deal is created on a different lane. - -Note that payment channels and vouchers can be used for any situation in which two parties need to incrementally transfer value between each other off-chain. - -## Using the CLI - -For example a client creates a payment channel to a provider with value 10 FIL. - -```sh -$ lotus paych add-funds 10 - -``` - -The client creates a voucher in lane 0 (implied) with nonce 1 (implied) and value 2. - -```sh -$ lotus paych voucher create 2 - -``` - -The client sends the voucher to the provider and the provider adds the voucher to their local store. - -```sh -$ lotus paych voucher add -``` - -The provider sends some data to the client. - -The client creates a voucher in lane 0 (implied) with nonce 2 (implied) and value 4. - -```sh -$ lotus paych voucher create 4 - -``` - -The client sends the voucher to the provider and the provider adds the voucher and sends back more data. -etc. - -The client can add value to the channel after it has been created by calling `paych add-funds` with the same client and provider addresses. - -```sh -$ lotus paych add-funds 5 - # Same address as above. Channel now has 15 -``` - -Once the client has received all their data, they may settle the channel. -Note that settlement doesn't have to be done immediately. -For example the client may keep the channel open as long as it wants to continue making deals with the provider. - -```sh -$ lotus paych settle -``` - -The provider can submit vouchers to chain (note that lotus does this automatically when it sees a settle message appear on chain). -The provider may have received many vouchers with incrementally higher values. -The provider should submit the best vouchers. Note that there will be one best voucher for each lane. - -```sh -$ lotus paych voucher best-spendable - - - - -$ lotus paych voucher submit -``` - -Once the settlement period is over, either the client or provider can call collect to disburse funds. - -```sh -$ lotus paych collect -``` - -Check the status of a channel that is still being created using `lotus paych status-by-from-to`. - -```sh -$ lotus paych status-by-from-to -Creating channel - From: t3sb6xzvs6rhlziatagevxpp3dwapdolurtkpn4kyh3kgoo4tn5o7lutjqlsnvpceztlhxu3lzzfe34rvpsjgq - To: t1zip4sblhyrn4oxygzsm6nafbsynp2avmk3xafea - Pending Amt: 10000 - Wait Sentinel: bafy2bzacedk2jidsyxcynusted35t5ipkhu2kpiodtwyjr3pimrhke6f5pqbm -``` - -Check the status of a channel that has been created using `lotus paych status`. - -```sh -$ lotus paych status -Channel exists - Channel: t2nydpzhmeqkmid5smtqnowlr2mr5az6rexpmyv6i - From: t3sb6xzvs6rhlziatagevxpp3dwapdolurtkpn4kyh3kgoo4tn5o7lutjqlsnvpceztlhxu3lzzfe34rvpsjgq - To: t1zip4sblhyrn4oxygzsm6nafbsynp2avmk3xafea - Confirmed Amt: 10000 - Pending Amt: 6000 - Queued Amt: 3000 - Voucher Redeemed Amt: 2000 -``` diff --git a/documentation/en/building/remote-api.md b/documentation/en/building/remote-api.md deleted file mode 100644 index d0fedb51b..000000000 --- a/documentation/en/building/remote-api.md +++ /dev/null @@ -1,69 +0,0 @@ -# Setting up remote API access - -The **Lotus Miner** and the **Lotus Node** applications come with their own local API endpoints setup by default when they are running. - -These endpoints are used by `lotus` and `lotus-miner` to interact with the running process. In this section we will explain how to enable remote access to the Lotus APIs. - -Note that instructions are the same for `lotus` and `lotus-miner`. For simplicity, we will just show how to do it with `lotus`. - -## Setting the listening interface for the API endpoint - -By default, the API listens on the local "loopback" interface (`127.0.0.1`). This is configured in the `config.toml` file: - -```toml -[API] -# ListenAddress = "/ip4/127.0.0.1/tcp/1234/http" -# RemoteListenAddress = "" -# Timeout = "30s" -``` - -To access the API remotely, Lotus needs to listen on the right IP/interface. The IP associated to each interface can be usually found with the command `ip a`. Once the right IP is known, it can be set in the configuration: - -```toml -[API] -ListenAddress = "/ip4//tcp/3453/http" # port is an example - -# Only relevant for lotus-miner -# This should be the IP:Port pair where the miner is reachable from anyone trying to dial to it. -# If you have placed a reverse proxy or a NAT'ing device in front of it, this may be different from -# the EXTERNAL_INTERFACE_IP. -RemoteListenAddress = "" -``` - -> `0.0.0.0` can be used too. This is a wildcard that means "all interfaces". Depending on the network setup, this may affect security (listening on the wrong, exposed interface). - -After making these changes, please restart the affected process. - -## Issuing tokens - -Any client wishing to talk to the API endpoints will need a token. Tokens can be generated with: - -```sh -lotus auth create-token --perm -``` - -(similarly for the Lotus Miner). - -The permissions work as follows: - -- `read` - Read node state, no private data. -- `write` - Write to local store / chain, and `read` permissions. -- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions. -- `admin` - Manage permissions, `read`, `write`, and `sign` permissions. - - -Tokens can then be used in applications by setting an Authorization header as: - -``` -Authorization: Bearer -``` - - -## Environment variables - -`lotus`, `lotus-miner` and `lotus-worker` can actually interact with their respective applications running on a different node. All is needed to configure them are the following the *environment variables*: - -```sh -FULLNODE_API_INFO="TOKEN:/ip4//tcp//http" -MINER_API_INFO="TOKEN:/ip4//tcp//http" -``` diff --git a/documentation/en/getting-started/getting-started.md b/documentation/en/getting-started/getting-started.md deleted file mode 100644 index 99b4095d4..000000000 --- a/documentation/en/getting-started/getting-started.md +++ /dev/null @@ -1,3 +0,0 @@ -# Getting started - -This section will get you started with Lotus. We will setup the Lotus daemon (that should already be [installed](en+install)), start it, create a wallet and use it to send and receive some Filecoin. diff --git a/documentation/en/getting-started/setup-troubleshooting.md b/documentation/en/getting-started/setup-troubleshooting.md deleted file mode 100644 index f27a3faa5..000000000 --- a/documentation/en/getting-started/setup-troubleshooting.md +++ /dev/null @@ -1,57 +0,0 @@ -# Setup Troubleshooting - - -## Error: initializing node error: cbor input had wrong number of fields - -This happens when you are starting Lotus which has been compiled for one network, but it encounters data in the Lotus data folder which is for a different network, or for an older incompatible version. - -The solution is to clear the data folder (see below). - -## Config: Clearing data - -Here is a command that will delete your chain data, stored wallets, stored data and any miners you have set up: - -```sh -rm -rf ~/.lotus ~/.lotusminer -``` - -Note you do not always need to clear your data for [updating](en+update). - -## Error: Failed to connect bootstrap peer - -```sh -WARN peermgr peermgr/peermgr.go:131 failed to connect to bootstrap peer: failed to dial : all dials failed - * [/ip4/147.75.80.17/tcp/1347] failed to negotiate security protocol: connected to wrong peer -``` - -- Try running the build steps again and make sure that you have the latest code from GitHub. - -```sh -ERROR hello hello/hello.go:81 other peer has different genesis! -``` - -- Try deleting your file system's `~/.lotus` directory. Check that it exists with `ls ~/.lotus`. - -```sh -- repo is already locked -``` - -- You already have another lotus daemon running. - -## Config: Open files limit - -Lotus will attempt to set up the file descriptor (FD) limit automatically. If that does not work, you can still configure your system to allow higher than the default values. - -On most systems you can check the open files limit with: - -```sh -ulimit -n -``` - -You can also modify this number by using the `ulimit` command. It gives you the ability to control the resources available for the shell or process started by it. If the number is below 10000, you can change it with the following command prior to starting the Lotus daemon: - -```sh -ulimit -n 10000 -``` - -Note that this is not persisted and that systemd manages its own FD limits for services. Please use your favourite search engine to find instructions on how to persist and configure FD limits for your system. diff --git a/documentation/en/getting-started/setup.md b/documentation/en/getting-started/setup.md deleted file mode 100644 index e751da80b..000000000 --- a/documentation/en/getting-started/setup.md +++ /dev/null @@ -1,169 +0,0 @@ -# Setting up Lotus - -Your Lotus binaries have been installed and you are ready to start participating in the Filecoin network. - -## Selecting the right network - -You should have built the Lotus binaries from the right Github branch and Lotus will be fully setup to join the matching [Filecoin network](https://docs.filecoin.io/how-to/networks/). For more information on switching networks, check the [updating Lotus section](en+update). - -## Starting the daemon - -To start the daemon simply run: - -```sh -lotus daemon -``` - -or if you are using the provided systemd service files, do: - -```sh -systemctl start lotus-daemon -``` - -__If you are using Lotus from China__, make sure you set the following environment variable before running Lotus: - -``` -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - - -During the first start, Lotus: - -* Will setup its data folder at `~/.lotus` -* Will download the necessary parameters -* Start syncing the Lotus chain - -If you started lotus using systemd, the logs will appear in `/var/log/lotus/daemon.log` (not in journalctl as usual), otherwise you will see them in your screen. - -Do not be appalled by the amount of warnings and sometimes errors showing in the logs, there are usually part of the usual functioning of the daemon as part of a distributed network. - -## Waiting to sync - -After the first start, the chain will start syncing until it has reached the tip. You can check how far the syncing process is with: - -```sh -lotus sync status -``` - -You can also interactively wait for the chain to be fully synced with: - -```sh -lotus sync wait -``` - -## Interacting with the Lotus daemon - -As shown above, the `lotus` command allows to interact with the running daemon. You will see it getting used in many of the documentation examples. - -This command-line-interface is self-documenting: - -```sh -# Show general help -lotus --help -# Show specific help for the "client" subcommand -lotus client --help -``` - -For example, after your Lotus daemon has been running for a few minutes, use `lotus` to check the number of other peers that it is connected to in the Filecoin network: - -```sh -lotus net peers -``` - -## Controlling the logging level - -```sh -lotus log set-level -``` -This command can be used to toggle the logging levels of the different -systems of a Lotus node. In decreasing order -of logging detail, the levels are `debug`, `info`, `warn`, and `error`. - -As an example, -to set the `chain` and `blocksync` to log at the `debug` level, run -`lotus log set-level --system chain --system blocksync debug`. - -To see the various logging system, run `lotus log list`. - - -## Configuration - -### Configuration file - -The Lotus daemon stores a configuration file in `~/.lotus/config.toml`. Note that by default all settings are commented. Here is an example configuration: - -```toml -[API] - # Binding address for the Lotus API - ListenAddress = "/ip4/127.0.0.1/tcp/1234/http" - # Not used by lotus daemon - RemoteListenAddress = "" - # General network timeout value - Timeout = "30s" - -# Libp2p provides connectivity to other Filecoin network nodes -[Libp2p] - # Binding address swarm - 0 means random port. - ListenAddresses = ["/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0"] - # Insert any addresses you want to explicitally - # announce to other peers here. Otherwise, they are - # guessed. - AnnounceAddresses = [] - # Insert any addresses to avoid announcing here. - NoAnnounceAddresses = [] - # Connection manager settings, decrease if your - # machine is overwhelmed by connections. - ConnMgrLow = 150 - ConnMgrHigh = 180 - ConnMgrGrace = "20s" - -# Pubsub is used to broadcast information in the network -[Pubsub] - Bootstrapper = false - RemoteTracer = "/dns4/pubsub-tracer.filecoin.io/tcp/4001/p2p/QmTd6UvR47vUidRNZ1ZKXHrAFhqTJAD27rKL9XYghEKgKX" - -# This section can be used to enable adding and retriving files from IPFS -[Client] - UseIpfs = false - IpfsMAddr = "" - IpfsUseForRetrieval = false - -# Metrics configuration -[Metrics] - Nickname = "" - HeadNotifs = false -``` - -### Ensuring connectivity to your Lotus daemon - -Usually your lotus daemon will establish connectivity with others in the network and try to make itself diallable using uPnP. If you wish to manually ensure that your daemon is reachable: - -* Set a fixed port of your choice in the `ListenAddresses` in the Libp2p section (i.e. 6665). -* Open a port in your router that is forwarded to this port. This is usually called featured as "Port forwarding" and the instructions differ from router model to model but there are many guides online. -* Add your public IP/port to `AnnounceAddresses`. i.e. `/ip4//tcp/6665/`. - -Note that it is not a requirement to use Lotus as a client to the network to be fully reachable, as your node already connects to others directly. - - -### Environment variables - -Common to most Lotus binaries: - -* `LOTUS_FD_MAX`: Sets the file descriptor limit for the process -* `LOTUS_JAEGER`: Sets the Jaeger URL to send traces. See TODO. -* `LOTUS_DEV`: Any non-empty value will enable more verbose logging, useful only for developers. - -Specific to the *Lotus daemon*: - -* `LOTUS_PATH`: Location to store Lotus data (defaults to `~/.lotus`). -* `LOTUS_SKIP_GENESIS_CHECK=_yes_`: Set only if you wish to run a lotus network with a different genesis block. -* `LOTUS_CHAIN_TIPSET_CACHE`: Sets the size for the chainstore tipset cache. Defaults to `8192`. Increase if you perform frequent arbitrary tipset lookups. -* `LOTUS_CHAIN_INDEX_CACHE`: Sets the size for the epoch index cache. Defaults to `32768`. Increase if you perform frequent deep chain lookups for block heights far from the latest height. -* `LOTUS_BSYNC_MSG_WINDOW`: Set the initial maximum window size for message fetching blocksync request. Set to 10-20 if you have an internet connection with low bandwidth. - -Specific to the *Lotus miner*: - -* `LOTUS_MINER_PATH`: Location for the miner's on-disk repo. Defaults to `./lotusminer`. -* A number of environment variables are respected for configuring the behaviour of the Filecoin proving subsystem. [See here](en+miner-setup). - - diff --git a/documentation/en/getting-started/wallet.md b/documentation/en/getting-started/wallet.md deleted file mode 100644 index 25a67fb09..000000000 --- a/documentation/en/getting-started/wallet.md +++ /dev/null @@ -1,58 +0,0 @@ -# Obtaining and sending FIL - -In order to receive and send FIL with Lotus you will need to have installed the program and be running the Lotus daemon. - -## Creating a wallet - - -```sh -lotus wallet new bls -``` - -This will print your Filecoin address. - -Your wallet information is stored in the `~/.lotus/keystore` (or `$LOTUS_PATH/keystore`). For instructions on export/import, see below. - -You can create multiple wallets and list them with: - -```sh -lotus wallet list -``` - -## Obtaining FIL - -FIL can be obtained either by using one of the Faucets (available for the test networks) or by buying it from an exchange supporting FIL trading (once mainnet has launched). - -Once you have received some FIL you can check your balance with: - -```sh -lotus wallet balance -``` - -Remember that your will only see the latest balance when your daemon is fully synced to the chain. - -## Sending FIL - -Sending some FIL can be achieved by running: - -```sh -lotus wallet send
-``` - -Make sure to check `lotus wallet send --help` for additional options. - -## Exporting and importing a wallet - -You can export and re-import a wallet with: - -```sh -lotus wallet export
> wallet.private -``` - -and: - -```sh -lotus wallet import wallet.private -``` - -Keep your wallet's private key safe! diff --git a/documentation/en/installation/install-linux.md b/documentation/en/installation/install-linux.md deleted file mode 100644 index 6fe12996e..000000000 --- a/documentation/en/installation/install-linux.md +++ /dev/null @@ -1,129 +0,0 @@ -# Linux installation - -This page will show you the steps to build and install Lotus in your Linux computer. - -## Dependencies - -### System dependencies - -First of all, building Lotus will require installing some system dependencies, usually provided by your distribution. - -For Arch Linux: - -```sh -sudo pacman -Syu opencl-icd-loader gcc git bzr jq pkg-config opencl-icd-loader opencl-headers -``` - -For Ubuntu: - -```sh -sudo apt update -sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl -sudo apt upgrade -``` - -For Fedora: - -```sh -sudo dnf -y update -sudo dnf -y install gcc git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm -``` - -For OpenSUSE: - -```sh -sudo zypper in gcc git jq make libOpenCL1 opencl-headers ocl-icd-devel clang llvm -sudo ln -s /usr/lib64/libOpenCL.so.1 /usr/lib64/libOpenCL.so -``` - -### Rustup - -Lotus needs [rustup](https://rustup.rs/): - -```sh -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -``` - -Please make sure your `$PATH` variable is correctly configured after the rustup installation so that `cargo` and `rustc` are found in their rustup-configured locations. - -### Go - -To build lotus you will need a working installation of **[Go1.14](https://golang.org/dl/)**. Follow the [installation instructions](https://golang.org/doc/install), which generally amount to: - -```sh -# Example! Check the installation instructions. -wget -c https://dl.google.com/go/go1.14.7.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local -``` - -## Build and install Lotus - -With all the above, you are ready to build and install the Lotus suite (`lotus`, `lotus-miner` and `lotus-worker`): - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -__IF YOU ARE IN CHINA__, set `export GOPROXY=https://goproxy.cn` before building - -Now, choose the network that you will be joining: - -* For `testnet`: `git checkout master` -* For `nerpa`: `git checkout ntwk-nerpa` -* For `butterfly`: `git checkout ntwk-butterfly` - -Once on the right branch, do: - -```sh -make clean install -sudo make install -``` - -This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`. `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets...). `lotus-miner` will use `$HOME/.lotusminer` respectively. See the *environment variables* section below for how to customize these. - -> Remeber to [move your Lotus folder](en+update) if you are switching between different networks, or there has been a network reset. - - -### Native Filecoin FFI - -Some newer processors (AMD Zen (and later), Intel Ice Lake) have support SHA extensions. To make full use of your processor's capabilities, make sure you set the following variables BEFORE building from source (as described above): - -```sh -export RUSTFLAGS="-C target-cpu=native -g" -export FFI_BUILD_FROM_SOURCE=1 -``` - -> __NOTE__: This method of building does not produce portable binaries! Make sure you run the binary in the same machine as you built it. - -### systemd service files - -Lotus provides Systemd service files. They can be installed with: - -```sh -make install-daemon-service -make install-miner-service -``` - -After that, you should be able to control Lotus using `systemctl`. - -## Troubleshooting - -This section mentions some of the common pitfalls for building Lotus. Check the [getting started](en+getting-started) section for more tips on issues when running the lotus daemon. - -### Build errors - -Please check the build logs closely. If you have a dirty state in your git branch make sure to: - -```sh -git checkout -git reset origin/ --hard -make clean -``` - -### Slow builds from China - -Users from China can speed up their builds by setting: - -```sh -export GOPROXY=https://goproxy.cn -``` diff --git a/documentation/en/installation/install-macos.md b/documentation/en/installation/install-macos.md deleted file mode 100644 index ea9ecb8ca..000000000 --- a/documentation/en/installation/install-macos.md +++ /dev/null @@ -1,62 +0,0 @@ -# MacOS Instructions - -## Get XCode Command Line Tools - -To check if you already have the XCode Command Line Tools installed via the CLI, run: - -```sh -xcode-select -p -``` - -If this command returns a path, you can move on to the next step. Otherwise, to install via the CLI, run: - -```sh -xcode-select --install -``` - -To update, run: - -```sh -sudo rm -rf /Library/Developer/CommandLineTools -xcode-select --install -``` - -## Get HomeBrew - -We recommend that MacOS users use [HomeBrew](https://brew.sh) to install each the necessary packages. - -Check if you have HomeBrew: - -```sh -brew -v -``` - -This command returns a version number if you have HomeBrew installed and nothing otherwise. - -In your terminal, enter this command to install Homebrew: - -```sh -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -``` - -Use the command `brew install` to install the following packages: - -```sh -brew install go bzr jq pkg-config rustup -``` - -Clone - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -Build - -```sh -make clean && make all -sudo make install -``` - -After intalling Lotus you will be ready to [setup and run the daemon](en+setup.md). diff --git a/documentation/en/installation/installation.md b/documentation/en/installation/installation.md deleted file mode 100644 index 98534da92..000000000 --- a/documentation/en/installation/installation.md +++ /dev/null @@ -1,39 +0,0 @@ -# Installation - -Lotus can be installed in [Linux](en-install-linux) and [MacOS](en-install-macos) machines by building it from source. Windows is not supported yet. - -This section contains guides to install Lotus in the supported platforms. - -Lotus is made of 3 binaries: - -* `lotus`: the main [Lotus node](en+setup) (Lotus client) -* `lotus-miner`: an application specifically for [Filecoin mining](en+miner-setup) -* `lotus-worker`: an additional [application to offload some heavy-processing tasks](en+lotus-worker) from the Lotus Miner. - -These applications are written in Go, but also import several Rust libraries. Lotus does not distribute -pre-compiled builds. - -## Hardware requirements - -### For client nodes - -* 8GiB of RAM -* Recommended for syncing speed: CPU with support for *Intel SHA Extensions* (AMD since Zen microarchitecture, Intel since Ice Lake). -* Recommended for speed: SSD hard drive (the bigger the better) - -### For miners - -The following correspond to the latest testing configuration: - -* 2 TB of hard drive space -* 8 core CPU -* 128 GiB of RAM with 256 GiB of NVMe SSD storage for swap (or simply, more RAM). -* Recommended for speed: CPU with support for *Intel SHA Extensions* (AMD since Zen microarchitecture, Intel since Ice Lake). -* GPU for block mining. The following have been [confirmed to be fast enough](en+gpus): - -- GeForce RTX 2080 Ti -- GeForce RTX 2080 SUPER -- GeForce RTX 2080 -- GeForce GTX 1080 Ti -- GeForce GTX 1080 -- GeForce GTX 1060 diff --git a/documentation/en/installation/update.md b/documentation/en/installation/update.md deleted file mode 100644 index 5d76592c9..000000000 --- a/documentation/en/installation/update.md +++ /dev/null @@ -1,72 +0,0 @@ -# Updating and restarting Lotus - -Updating Lotus is as simple as rebuilding and re-installing the software as explained in the previous sections. - -You can verify which version of Lotus you are running with: - -```sh -lotus version -``` - -Make sure that you `git pull` the branch that corresponds to the network that your Lotus daemon is using: - -```sh -git pull origin -make clean -make all -sudo make install # if necessary -``` - -Finally, restart the Lotus Node and/or Lotus Miner(s). - -__CAVEAT__: If you are running miners: check if your miner is safe to shut down and restart: `lotus-miner proving info`. If any deadline shows a block height in the past, do not restart: - -In the following example, Deadline Open is 454 which is earlier than Current Epoch of 500. This miner should **not** be shut down or restarted. - -``` -$ sudo lotus-miner proving info -Miner: t01001 -Current Epoch: 500 -Proving Period Boundary: 154 -Proving Period Start: 154 (2h53m0s ago) -Next Period Start: 3034 (in 21h7m0s) -Faults: 768 (100.00%) -Recovering: 768 -Deadline Index: 5 -Deadline Sectors: 0 -Deadline Open: 454 (23m0s ago) -Deadline Close: 514 (in 7m0s) -Deadline Challenge: 434 (33m0s ago) -Deadline FaultCutoff: 384 (58m0s ago) -``` - -In this next example, the miner can be safely restarted because no Deadlines are earlier than Current Epoch of 497. You have ~45 minutes before the miner must be back online to declare faults (FaultCutoff). If the miner has no faults, you have about an hour. - -``` -$ sudo lotus-miner proving info -Miner: t01000 -Current Epoch: 497 -Proving Period Boundary: 658 -Proving Period Start: 658 (in 1h20m30s) -Next Period Start: 3538 (in 25h20m30s) -Faults: 0 (0.00%) -Recovering: 0 -Deadline Index: 0 -Deadline Sectors: 768 -Deadline Open: 658 (in 1h20m30s) -Deadline Close: 718 (in 1h50m30s) -Deadline Challenge: 638 (in 1h10m30s) -Deadline FaultCutoff: 588 (in 45m30s) -``` - -## Switching networks and network resets - -If you wish to switch to a different lotus network or there has been a network reset, you will need to: - -* Checkout the appropiate repository branch and rebuild -* Ensure you do not mix Lotus data (`LOTUS_PATH`, usually `~/.lotus`) from a previous or different network. For this, either: - * Rename the folder to something else or, - * Set a different `LOTUS_PATH` for the new network. -* Same for `~/.lotusminer` if you are running a miner. - -Note that deleting the Lotus data folder will wipe all the chain data, wallets and configuration, so think twice before taking any non-reversible action. diff --git a/documentation/en/mining/gpus.md b/documentation/en/mining/gpus.md deleted file mode 100644 index ad0ed4f66..000000000 --- a/documentation/en/mining/gpus.md +++ /dev/null @@ -1,17 +0,0 @@ -# Benchmarking additional GPUs - -If you want to test a GPU that is not explicitly supported, set the following *environment variable*: - -```sh -BELLMAN_CUSTOM_GPU=":" -``` - -Here is an example of trying a GeForce GTX 1660 Ti with 1536 cores. - -```sh -BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536" -``` - -To get the number of cores for your GPU, you will need to check your card’s specifications. - -To perform the benchmark you can use Lotus' [benchmarking tool](https://github.com/filecoin-project/lotus/tree/master/cmd/lotus-bench). Results and discussion are tracked in a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694). diff --git a/documentation/en/mining/lotus-seal-worker.md b/documentation/en/mining/lotus-seal-worker.md deleted file mode 100644 index 47e201ca5..000000000 --- a/documentation/en/mining/lotus-seal-worker.md +++ /dev/null @@ -1,99 +0,0 @@ -# Lotus Worker - -The **Lotus Worker** is an extra process that can offload heavy processing tasks from your **Lotus Miner**. The sealing process automatically runs in the **Lotus Miner** process, but you can use the Worker on another machine communicating over a fast network to free up resources on the machine running the mining process. - -## Installation - -The `lotus-worker` application is installed along with the others when running `sudo make install` as shown in the [Installation section](en+install-linux). For simplicity, we recommend following the same procedure in the machines that will run the Lotus Workers (even if the Lotus miner and the Lotus daemon are not used there). - -## Setting up the Miner - -### Allow external connections to the miner API - -First, you will need to ensure your `lotus-miner`'s API is accessible over the network. - -To do this, open up `~/.lotusminer/config.toml` (Or if you manually set `LOTUS_MINER_PATH`, look under that directory) and look for the API field. - -Default config: - -```toml -[API] -ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" -RemoteListenAddress = "127.0.0.1:2345" -``` - -To make your node accessible over the local area network, you will need to determine your machine's IP on the LAN (`ip a`), and change the `127.0.0.1` in the file to that address. - -A more permissive and less secure option is to change it to `0.0.0.0`. This will allow anyone who can connect to your computer on that port to access the miner's API, though they will still need an auth token. - -`RemoteListenAddress` must be set to an address which other nodes on your network will be able to reach. - -### Create an authentication token - -Write down the output of: - -```sh -lotus-miner auth api-info --perm admin -``` - -The Lotus Workers will need this token to connect to the miner. - -## Connecting the Lotus Workers - -On each machine that will run the `lotus-worker` application you will need to define the following *environment variable*: - -```sh -export MINER_API_INFO::/ip4//tcp/2345` -``` - -If you are trying to use `lotus-worker` from China. You should additionally set: - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - - -Once that is done, you can run the Worker with: - -```sh -lotus-worker run -``` - -> If you are running multiple workers on the same host, you will need to specify the `--listen` flag and ensure each worker is on a different port. - -On your Lotus miner, check that the workers are correctly connected: - -```sh -lotus-miner sealing workers -Worker 0, host computer - CPU: [ ] 0 core(s) in use - RAM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - VMEM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - GPU: GeForce RTX 2080, not used - -Worker 1, host othercomputer - CPU: [ ] 0 core(s) in use - RAM: [|||||||||||||| ] 23% 14 GiB/62.7 GiB - VMEM: [|||||||||||||| ] 23% 14 GiB/62.7 GiB - GPU: GeForce RTX 2080, not used -``` - -## Running locally for manually managing process priority - -You can also run the **Lotus Worker** on the same machine as your **Lotus Miner**, so you can manually manage the process priority. - -To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes: - -```toml -[Storage] - AllowPreCommit1 = false - AllowPreCommit2 = false - AllowCommit = false - AllowUnseal = false -``` - -You can then run the miner on your local-loopback interface; - -```sh -lotus-worker run -``` diff --git a/documentation/en/mining/managing-deals.md b/documentation/en/mining/managing-deals.md deleted file mode 100644 index 5f73a6a2d..000000000 --- a/documentation/en/mining/managing-deals.md +++ /dev/null @@ -1,19 +0,0 @@ -# Managing deals - - -While the Lotus Miner is running as a daemon, the `lotus-miner` application can be used to manage and configure the miner: - - -```sh -lotus-miner storage-deals --help -``` - -Running the above command will show the different options related to deals. For example, `lotus-miner storage-deals set-ask` allows to set the price for storage that your miner uses to respond ask requests from clients. - -If deals are ongoing, you can check the data transfers with: - -```sh -lotus-miner data-transfers list -``` - -Make sure you explore the `lotus-miner` CLI. Every command is self-documented and takes a `--help` flag that offers specific information about it. diff --git a/documentation/en/mining/miner-setup.md b/documentation/en/mining/miner-setup.md deleted file mode 100644 index cafa1e7b1..000000000 --- a/documentation/en/mining/miner-setup.md +++ /dev/null @@ -1,241 +0,0 @@ -# Miner setup - -This page will guide you through all you need to know to sucessfully run a **Lotus Miner**. Before proceeding, remember that you should be running the Lotus daemon on a fully synced chain. - -## Performance tweaks - -This is a list of performance tweaks to consider before starting the miner: - -### Building - -As [explained already](en+install-linux#native-filecoin-ffi-10) should have exported the following variables before building the Lotus applications: - -```sh -export RUSTFLAGS="-C target-cpu=native -g" -export FFI_BUILD_FROM_SOURCE=1 -``` - -### Environment - -For high performance mining, we recommend setting the following variables in your environment so that they are available when running any of the Lotus applications: - -```sh -# See https://github.com/filecoin-project/bellman -export BELLMAN_CPU_UTILIZATION=0.875 - -# See https://github.com/filecoin-project/rust-fil-proofs/ -export FIL_PROOFS_MAXIMIZE_CACHING=1 # More speed at RAM cost (1x sector-size of RAM - 32 GB). -export FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1 # precommit2 GPU acceleration -export FIL_PROOFS_USE_GPU_TREE_BUILDER=1 -``` - -IF YOU ARE RUNNING FROM CHINA: - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - -IF YOUR MINER RUNS IN A DIFFERENT MACHINE AS THE LOTUS DAEMON: - -```sh -export FULLNODE_API_INFO=:/ip4//tcp//http -``` - -If you will be using systemd service files to run the Lotus daemon and miner, make sure you include these variables manually in the service files. - -### Adding swap - -If you have only 128GiB of RAM, you will need to make sure your system provides at least an extra 256GiB of fast swap (preferably NVMe SSD): - -```sh -sudo fallocate -l 256G /swapfile -sudo chmod 600 /swapfile -sudo mkswap /swapfile -sudo swapon /swapfile -# show current swap spaces and take note of the current highest priority -swapon --show -# append the following line to /etc/fstab (ensure highest priority) and then reboot -# /swapfile swap swap pri=50 0 0 -sudo reboot -# check a 256GB swap file is automatically mounted and has the highest priority -swapon --show -``` - -## Creating a new BLS wallet - -You will need a BLS wallet (`t3...`) for mining. To create it, if you don't have one already, run: - -```sh -lotus wallet new bls -``` - -Next make sure to [send some funds](en+wallet) to this address so that the miner setup can be completed. - -## Initializing the miner - -> SPACE RACE: -> To participate in the Space race, please register your miner: -> -> - Visit the [faucet](http://spacerace.faucet.glif.io/) -> - Paste the address you created under REQUEST. -> - Press the Request button. - -Now that you have a miner address you can initialize the Lotus Miner: - -```sh -lotus-miner init --owner= --no-local-storage -``` - -* The `--no-local-storage` flag is used so that we configure specific locations for storage later below. -* The init process will download over 100GiB of initialization parameters to /var/tmp/filecoin-proof-parameters. Make sure there is space or set `FIL_PROOFS_PARAMETER_CACHE` to somewhere else. -* The Lotus Miner configuration folder is created at `~/.lotusminer/` or `$LOTUS_MINER_PATH` if set. - -## Reachability - -Before you start your miner, it is __very important__ to configure it so that it is reachable from any peer in the Filecoin network. For this you will need a stable public IP and edit your `~/.lotusminer/config.toml` as follows: - -```toml -... -[Libp2p] - ListenAddresses = ["/ip4/0.0.0.0/tcp/24001"] # choose a fixed port - AnnounceAddresses = ["/ip4//tcp/24001"] # important! -... -``` - -Once you start your miner, make sure you can connect to its public IP/port (you can use `telnet`, `nc` for the task...). If you have an active firewall or some sort, you may need to additionally open ports in it. - - -## Starting the miner - -You are now ready to start your Lotus miner: - -```sh -lotus-miner run -``` - -or if you are using the systemd service file: - -```sh -systemctl start lotus-miner -``` - -> __Do not proceed__ from here until you have verified that your miner not only is running, but also __reachable on its public IP address__. - -## Publishing the miner addresses - -Once the miner is up and running, publish your miner address (which you configured above) on the chain (please ensure it is dialable): - -```sh -lotus-miner actor set-addrs /ip4//tcp/24001 -``` - -## Setting locations for sealing and long-term storage - -If you used the `--no-local-storage` flag during initialization, you can now specify the disk locations for sealing (SSD recommended) and long-term storage (otherwise you can skip this): - -``` -lotus-miner storage attach --init --seal -lotus-miner storage attach --init --store -lotus-miner storage list -``` - -## Pledging sectors - -If you would like to compete for block rewards by increasing your power in the network as soon as possible, you can optionally pledge one or several sectors, depending on your storage. It can also be used to test that the sealing process works correctly. Pledging is equivalent to storing random data instead of real data obtained through storage deals. - -> Note that pledging sectors to the mainnet network makes most sense when trying to obtain a reasonable amount of total power in the network, thus obtaining real chances to mine new blocks. Otherwise it is only useful for testing purposes. - -If you decide to go ahead, then do: - -```sh -lotus-miner sectors pledge -``` - -This will write data to `$TMPDIR` so make sure that there is enough space available. - -You shoud check that your sealing job has started with: - -```sh -lotus-miner sealing jobs -``` - -This will be accommpanied by a file in `/unsealed`. - -After some minutes, you can check the sealing progress with: - -```sh -lotus-miner sectors list -# and -lotus-miner sealing workers -``` - -When sealing for the new is complete, `pSet: NO` will become `pSet: YES`. - -Once the sealing is finished, you will want to configure how long it took your miner to seal this sector and configure the miner accordingly. To find out how long it took use: - -``` -lotus-miner sectors status --log 0 -``` - -Once you know, you can edit the Miner's `~/.lotusminer/config.toml` accordingly: - -``` -... -[Dealmaking] -... - ExpectedSealDuration = "12h0m0s" # The time it took your miner -``` - -You can also take the chance to edit other values, such as `WaitForDealsDelay` which specifies the delay between accepting the first deal and sealing, allowing to place multiple deals in the same sector. - -Once you are done editing the configuration, [restart your miner](en+update). - -If you wish to be able to re-use a pledged sector for real storage deals before the pledged period of 6 months ends, you will need to mark them for upgrade: - -```sh -lotus-miner sectors mark-for-upgrade -``` - -The sector should become inactive within 24 hours. From that point, the pledged storage can be re-used to store real data associated with real storage deals. - -## Separate address for windowPoSt messages - -WindowPoSt is the mechanism through which storage is verified in Filecoin. It requires miners to submit proofs for all sectors every 24h, which require sending messages to the chain. - -Because many other mining related actions require sending messages to the chain, and not all of those are "high value", it may be desirable to use a separate account to send PoSt messages from. This allows for setting lower GasFeeCaps on the lower value messages without creating head-of-line blocking problems for the PoSt messages in congested chain conditions - -To set this up, first create a new account, and send it some funds for gas fees: - -```sh -lotus wallet new bls -t3defg... - -lotus send t3defg... 100 -``` - -Next add the control address: - -```sh -lotus-miner actor control set --really-do-it t3defg... -Add t3defg... -Message CID: bafy2.. -``` - -Wait for the message to land on chain: - -```sh -lotus state wait-msg bafy2.. -... -Exit Code: 0 -... -``` - -Finally, check the miner control address list to make sure the address was correctly setup: - -```sh -lotus-miner actor control list -name ID key use balance -owner t01111 t3abcd... other 300 FIL -worker t01111 t3abcd... other 300 FIL -control-0 t02222 t3defg... post 100 FIL -``` diff --git a/documentation/en/mining/mining-troubleshooting.md b/documentation/en/mining/mining-troubleshooting.md deleted file mode 100644 index a9972c2bd..000000000 --- a/documentation/en/mining/mining-troubleshooting.md +++ /dev/null @@ -1,59 +0,0 @@ -# Mining Troubleshooting - -## Config: Filecoin Proof Parameters directory - -If you want to put the **Filecoin Proof Parameters** in a different directory, use the following environment variable: - -```sh -FIL_PROOFS_PARAMETER_CACHE -``` - -## Error: Can't acquire bellman.lock - -The **Bellman** lockfile is created to lock a GPU for a process. This bug can occur when this file isn't properly cleaned up: - -```sh -mining block failed: computing election proof: github.com/filecoin-project/lotus/miner.(*Miner).mineOne -``` - -This bug occurs when the miner can't acquire the `bellman.lock`. To fix it you need to stop the `lotus-miner` and remove `/tmp/bellman.lock`. - -## Error: Failed to get api endpoint - -```sh -lotus-miner info -# WARN main lotus-miner/main.go:73 failed to get api endpoint: (/Users/myrmidon/.lotusminer) %!w(*errors.errorString=&{API not running (no endpoint)}): -``` - -If you see this, that means your **Lotus Miner** isn't ready yet. You need to finish [syncing the chain](en+setup#waiting-to-sync-370). - -## Error: Your computer may not be fast enough - -```sh -CAUTION: block production took longer than the block delay. Your computer may not be fast enough to keep up -``` - -If you see this, that means your computer is too slow and your blocks are not included in the chain, and you will not receive any rewards. - -## Error: No space left on device - -```sh -lotus-miner sectors pledge -# No space left on device (os error 28) -``` - -If you see this, that means `pledge-sector` wrote too much data to `$TMPDIR` which by default is the root partition (This is common for Linux setups). Usually your root partition does not get the largest partition of storage so you will need to change the environment variable to something else. - -## Error: GPU unused - -If you suspect that your GPU is not being used, first make sure it is properly configured as described in the [testing configuration page](hardware-mining.md). Once you've done that (and set the `BELLMAN_CUSTOM_GPU` as appropriate if necessary) you can verify your GPU is being used by running a quick lotus-bench benchmark. - -First, to watch GPU utilization run `nvtop` in one terminal, then in a separate terminal, run: - -```sh -make bench -./bench sealing --sector-size=2KiB -``` - -This process uses a fair amount of GPU, and generally takes ~4 minutes to complete. If you do not see any activity in nvtop from lotus during the entire process, it is likely something is misconfigured with your GPU. - diff --git a/documentation/en/mining/mining.md b/documentation/en/mining/mining.md deleted file mode 100644 index b1b944c6e..000000000 --- a/documentation/en/mining/mining.md +++ /dev/null @@ -1,8 +0,0 @@ -# Storage Mining - -This section of the documentation explains how to do storage mining with Lotus. Please note that not everyone can do storage mining, and that you should not attempt it on on networks where sector sizes are 32GB+ unless you meet the [hardware requirements](en+install#hardware-requirements-1). - -From this point we assume that you have setup and are running the [Lotus Node](en+setup), that it has fully synced the Filecoin chain and that you are familiar with how to interact with it using the `lotus` command-line interface. - -In order to perform storage mining, apart from the Lotus daemon, you will be additionally interacting with the `lotus-miner` and potentially the `lotus-worker` applications (which you should have [installed](en+install-linux) along the `lotus` application already). - diff --git a/documentation/en/store/adding-from-ipfs.md b/documentation/en/store/adding-from-ipfs.md deleted file mode 100644 index 2f6b097cc..000000000 --- a/documentation/en/store/adding-from-ipfs.md +++ /dev/null @@ -1,20 +0,0 @@ -# Adding data from IPFS - -Lotus supports making deals with data stored in IPFS, without having to re-import it into lotus. - -To enable this integration, you need to have an IPFS daemon running in the background. - -Then, open up `~/.lotus/config.toml` (or if you manually set `LOTUS_PATH`, look under that directory) and look for the Client field, and set `UseIpfs` to `true`. - -```toml -[Client] -UseIpfs = true -``` - -After restarting the lotus daemon, you should be able to make deals with data in your IPFS node: - -```sh -$ ipfs add -r SomeData -QmSomeData -$ ./lotus client deal QmSomeData t01000 0.0000000001 80000 -``` diff --git a/documentation/en/store/making-deals.md b/documentation/en/store/making-deals.md deleted file mode 100644 index ca3a47182..000000000 --- a/documentation/en/store/making-deals.md +++ /dev/null @@ -1,71 +0,0 @@ -# Making storage deals - -## Adding a file to Lotus - -Before sending data to a Filecoin miner for storage, the data needs to be correctly formatted and packed. This can be achieved by locally importing the data into Lotus with: - -```sh -lotus client import ./your-example-file.txt -``` - -Upon success, this command will return a **Data CID**. This is a very important piece of information, as it will be used to make deals to both store and retrieve the data in the future. - -You can list the data CIDs of the files you locally imported with: - -```sh -lotus client local -``` - -## Storing data in the network - -To store data in the network you will need to: - -* Find a Filecoin miner willing to store it -* Make a deal with the miner agreeing on the price to pay and the duration for which the data should be stored. - -You can obtain a list of all miners in the network with: - -```sh -lotus state list-miners -t0xxxx -t0xxxy -t0xxxz -... -``` - -This will print a list of miner IDs. In order to ask for the terms offered by a particular miner, you can then run: - -```sh -lotus client query-ask -``` - -If you are satisfied with the terms, you can proceed to propose a deal to the miner, using the **Data CID** that you obtained during the import step: - - -```sh -lotus client deal -``` - -This command will interactively ask you for the CID, miner ID and duration in days for the deal. You can also call it with arguments: - -```sh -lotus client deal -``` - -where the `duration` is expressed in blocks (1 block is equivalent to 30s). - -## Checking the status of the deals - -You can list deals with: - -```sh -lotus client list-deals -``` - -Among other things, this will give you information about the current state on your deals, whether they have been published on chain (by the miners) and whether the miners have been slashed for not honoring them. - -For a deal to succeed, the miner needs to be correctly configured and running, accept the deal and *seal* the file correctly. Otherwise, the deal will appear in error state. - -You can make deals with multiple miners for the same data. - -Once a deal is sucessful and the data is *sealed*, it can be [retrieved](en+retrieving). diff --git a/documentation/en/store/retrieve.md b/documentation/en/store/retrieve.md deleted file mode 100644 index 1e8db65af..000000000 --- a/documentation/en/store/retrieve.md +++ /dev/null @@ -1,27 +0,0 @@ -# Retrieving Data - -Once data has been succesfully [stored](en+making-deals) and sealed by a Filecoin miner, it can be retrieved. - -In order to do this we will need to create a **retrieval deal**. - -## Finding data by CID - -In order to retrieve some data you will need the **Data CID** that was used to create the storage deal. - -You can find who is storing the data by running: - -```sh -lotus client find -``` - -## Making a retrieval deal - -You can then make a retrieval deal with: - -```sh -lotus client retrieve -``` - -This commands take other optional flags (check `--help`). - -If the outfile does not exist it will be created in the Lotus repository directory. This process may take 2 to 10 minutes. diff --git a/documentation/en/store/storage-troubleshooting.md b/documentation/en/store/storage-troubleshooting.md deleted file mode 100644 index 7087ec3d0..000000000 --- a/documentation/en/store/storage-troubleshooting.md +++ /dev/null @@ -1,30 +0,0 @@ -# Storage Troubleshooting - -## Error: Routing: not found - -``` -WARN main lotus/main.go:72 routing: not found -``` - -This error means that the miner is offline. - -## Error: Failed to start deal - -```sh -WARN main lotus/main.go:72 failed to start deal: computing commP failed: generating CommP: Piece must be at least 127 bytes -``` - -This error means that there is a minimum file size of 127 bytes. - -## Error: 0kb file response during retrieval - -This means that the file to be retrieved may have not yet been sealed and is thus, not retrievable yet. - -Miners can check sealing progress with this command: - -```sh -lotus-miner sectors list -``` - -When sealing is complete, `pSet: NO` will become `pSet: YES`. - diff --git a/documentation/en/store/store.md b/documentation/en/store/store.md deleted file mode 100644 index 205bd0e23..000000000 --- a/documentation/en/store/store.md +++ /dev/null @@ -1,11 +0,0 @@ -# Storing and retrieving data - -Lotus enables you to store any data on the Filecoin network and retrieve it later. This is achieved by making *deals* with miners. - -A *storage deal* specifies that a miner should store ceratin data for a previously agreed period and price. - -Once a deal is made, the data is then sent to the miners, which regularly proves that it is storing it. If they fail to do so, the miner is penalized (slashed). - -The data can be retrieved with a *retrieval deal*. - -This section explains how to use Lotus to [store](en+making-deals) and [retrieve](en+retrieving) data from the Filecoin network. From 591e32af487204f58d092015a8ce25439b884718 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:41:46 +0200 Subject: [PATCH 09/80] Remove circle_ci docs-check. --- .circleci/config.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index acd447f69..d8f149889 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,15 +346,6 @@ jobs: - run: git --no-pager diff - run: git --no-pager diff --quiet - docs-check: - executor: golang - steps: - - install-deps - - prepare - - run: make docsgen - - run: git --no-pager diff - - run: git --no-pager diff --quiet - lint: &lint description: | Run golangci-lint. @@ -424,7 +415,6 @@ workflows: - mod-tidy-check - gofmt - cbor-gen-check - - docs-check - test: codecov-upload: true test-suite-name: full From bc9232544c7f5d68c737b9d7887ae05f0232ff34 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:48:09 +0200 Subject: [PATCH 10/80] Remove faqs too --- documentation/en/faqs.md | 132 --------------------------------------- 1 file changed, 132 deletions(-) delete mode 100644 documentation/en/faqs.md diff --git a/documentation/en/faqs.md b/documentation/en/faqs.md deleted file mode 100644 index 74119a5b6..000000000 --- a/documentation/en/faqs.md +++ /dev/null @@ -1,132 +0,0 @@ -# Frequently Asked Questions - -Here are some FAQs concerning the Lotus implementation and participation in -Testnet. -For questions concerning the broader Filecoin project, please -go [here](https://filecoin.io/faqs/). - -## Introduction to Lotus - -### What is Lotus? - -Lotus is an implementation of the **Filecoin Distributed Storage Network**, written in Go. -It is designed to be modular and interoperable with any other implementation of the Filecoin Protocol. - -### What are the components of Lotus? - -Lotus is composed of two separate pieces that can talk to each other: - -The Lotus Node can sync the blockchain, validating all blocks, transfers, and deals -along the way. It can also facilitate the creation of new storage deals. If you are not -interested in providing your own storage to the network, and do not want to produce blocks -yourself, then the Lotus Node is all you need! - -The Lotus Miner does everything you need for the registration of storage, and the -production of new blocks. The Lotus Miner communicates with the network by talking -to a Lotus Node over the JSON-RPC API. - -## Setting up a Lotus Node - -### How do I set up a Lotus Node? - -Follow the instructions found [here](en+install) and [here](en+setup). - -### Where can I get the latest version of Lotus? - -Download the binary tagged as the `Latest Release` from the [Lotus Github repo](https://github.com/filecoin-project/lotus/releases) or checkout the `master` branch of the source repository. - -### What operating systems can Lotus run on? - -Lotus can build and run on most Linux and MacOS systems with [at least 8GB of RAM](en+install#hardware-requirements-1). Windows is not yet supported. - -### How can I update to the latest version of Lotus? - -To update Lotus, follow the instructions [here](en+update). - -### How do I prepare a fresh installation of Lotus? - -Stop the Lotus daemon, and delete all related files, including sealed and chain data by -running `rm ~/.lotus ~/.lotusminer`. - -Then, install Lotus afresh by following the instructions -found [here](en+install). - -### Can I configure where the node's config and data goes? - -Yes! The `LOTUS_PATH` variable sets the path for where the Lotus node's data is written. -The `LOTUS_MINER_PATH` variable does the same for miner-specific information. - -## Interacting with a Lotus Node - -### How can I communicate with a Lotus Node? - -Lotus Nodes have a command-line interface, as well as a JSON-RPC API. - -### What are the commands I can send using the command-line interface? - -The command-line interface is self-documenting, try running `lotus --help` from the `lotus` home -directory for more. - -### How can I send a request over the JSON-RPC API? - -Information on how to send a `cURL` request to the JSON-RPC API can be found -[here](en+api). - -### What are the requests I can send over the JSON-RPC API? - -Please have a look [here](en+api). - - -## The Test Network - -### What is Testnet? - -Testnet is a live network of Lotus Nodes run by the -community for testing purposes. - -### Is FIL on the Testnet worth anything? - -Nothing at all! - -### How can I see the status of Testnet? - -The [dashboard](https://stats.testnet.filecoin.io/) displays the status of the network as -well as a ton of other metrics you might find interesting. - -## Mining with a Lotus Node on Testnet - -### How do I get started mining with Lotus? - -Follow the instructions found [here](en+mining). - -### What are the minimum hardware requirements? - -An example test configuration, and minimum hardware requirements can be found -[here](en+install#hardware-requirements-8). - -Note that these might NOT be the minimum requirements for mining on Mainnet. - -### What are some GPUs that have been tested? - -See previous question. - -### Why is my GPU not being used when sealing a sector? - -Sealing a sector does not involve constant GPU operations. It's possible -that your GPU simply isn't necessary at the moment you checked. - -## Advanced questions - -### Is there a Docker image for lotus? - -Community-contributed Docker and Docker Compose examples are available -[here](https://github.com/filecoin-project/lotus/tree/master/tools/dockers/docker-examples). - -### How can I run two miners on the same machine? - -You can do so by changing the storage path variable for the second miner, e.g., -`LOTUS_MINER_PATH=~/.lotusminer2`. You will also need to make sure that no ports collide. - -### How do I setup my own local devnet? - -Follow the instructions found [here](en+local-devnet). From 784399738a49f282bc8002771e080d2eba5a33f9 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:50:37 +0200 Subject: [PATCH 11/80] Keep api-methods.md even if not part of the docs --- Makefile | 2 +- documentation/en/api-methods.md | 4567 +++++++++++++++++++++++++++++++ 2 files changed, 4568 insertions(+), 1 deletion(-) create mode 100644 documentation/en/api-methods.md diff --git a/Makefile b/Makefile index 245d8a9a0..56ab361ec 100644 --- a/Makefile +++ b/Makefile @@ -286,7 +286,7 @@ method-gen: gen: type-gen method-gen docsgen: - go run ./api/docgen > documentation/en/building/api-methods.md + go run ./api/docgen > documentation/en/api-methods.md print-%: @echo $*=$($*) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md new file mode 100644 index 000000000..2f3164bb7 --- /dev/null +++ b/documentation/en/api-methods.md @@ -0,0 +1,4567 @@ +# Groups +* [](#) + * [Closing](#Closing) + * [Shutdown](#Shutdown) + * [Version](#Version) +* [Auth](#Auth) + * [AuthNew](#AuthNew) + * [AuthVerify](#AuthVerify) +* [Beacon](#Beacon) + * [BeaconGetEntry](#BeaconGetEntry) +* [Chain](#Chain) + * [ChainExport](#ChainExport) + * [ChainGetBlock](#ChainGetBlock) + * [ChainGetBlockMessages](#ChainGetBlockMessages) + * [ChainGetGenesis](#ChainGetGenesis) + * [ChainGetMessage](#ChainGetMessage) + * [ChainGetNode](#ChainGetNode) + * [ChainGetParentMessages](#ChainGetParentMessages) + * [ChainGetParentReceipts](#ChainGetParentReceipts) + * [ChainGetPath](#ChainGetPath) + * [ChainGetRandomnessFromBeacon](#ChainGetRandomnessFromBeacon) + * [ChainGetRandomnessFromTickets](#ChainGetRandomnessFromTickets) + * [ChainGetTipSet](#ChainGetTipSet) + * [ChainGetTipSetByHeight](#ChainGetTipSetByHeight) + * [ChainHasObj](#ChainHasObj) + * [ChainHead](#ChainHead) + * [ChainNotify](#ChainNotify) + * [ChainReadObj](#ChainReadObj) + * [ChainSetHead](#ChainSetHead) + * [ChainStatObj](#ChainStatObj) + * [ChainTipSetWeight](#ChainTipSetWeight) +* [Client](#Client) + * [ClientCalcCommP](#ClientCalcCommP) + * [ClientDataTransferUpdates](#ClientDataTransferUpdates) + * [ClientDealSize](#ClientDealSize) + * [ClientFindData](#ClientFindData) + * [ClientGenCar](#ClientGenCar) + * [ClientGetDealInfo](#ClientGetDealInfo) + * [ClientGetDealUpdates](#ClientGetDealUpdates) + * [ClientHasLocal](#ClientHasLocal) + * [ClientImport](#ClientImport) + * [ClientListDataTransfers](#ClientListDataTransfers) + * [ClientListDeals](#ClientListDeals) + * [ClientListImports](#ClientListImports) + * [ClientMinerQueryOffer](#ClientMinerQueryOffer) + * [ClientQueryAsk](#ClientQueryAsk) + * [ClientRemoveImport](#ClientRemoveImport) + * [ClientRetrieve](#ClientRetrieve) + * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) + * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) + * [ClientStartDeal](#ClientStartDeal) +* [Gas](#Gas) + * [GasEstimateFeeCap](#GasEstimateFeeCap) + * [GasEstimateGasLimit](#GasEstimateGasLimit) + * [GasEstimateGasPremium](#GasEstimateGasPremium) + * [GasEstimateMessageGas](#GasEstimateMessageGas) +* [I](#I) + * [ID](#ID) +* [Log](#Log) + * [LogList](#LogList) + * [LogSetLevel](#LogSetLevel) +* [Market](#Market) + * [MarketEnsureAvailable](#MarketEnsureAvailable) +* [Miner](#Miner) + * [MinerCreateBlock](#MinerCreateBlock) + * [MinerGetBaseInfo](#MinerGetBaseInfo) +* [Mpool](#Mpool) + * [MpoolClear](#MpoolClear) + * [MpoolGetConfig](#MpoolGetConfig) + * [MpoolGetNonce](#MpoolGetNonce) + * [MpoolPending](#MpoolPending) + * [MpoolPush](#MpoolPush) + * [MpoolPushMessage](#MpoolPushMessage) + * [MpoolSelect](#MpoolSelect) + * [MpoolSetConfig](#MpoolSetConfig) + * [MpoolSub](#MpoolSub) +* [Msig](#Msig) + * [MsigAddApprove](#MsigAddApprove) + * [MsigAddCancel](#MsigAddCancel) + * [MsigAddPropose](#MsigAddPropose) + * [MsigApprove](#MsigApprove) + * [MsigCancel](#MsigCancel) + * [MsigCreate](#MsigCreate) + * [MsigGetAvailableBalance](#MsigGetAvailableBalance) + * [MsigGetVested](#MsigGetVested) + * [MsigPropose](#MsigPropose) + * [MsigSwapApprove](#MsigSwapApprove) + * [MsigSwapCancel](#MsigSwapCancel) + * [MsigSwapPropose](#MsigSwapPropose) +* [Net](#Net) + * [NetAddrsListen](#NetAddrsListen) + * [NetAgentVersion](#NetAgentVersion) + * [NetAutoNatStatus](#NetAutoNatStatus) + * [NetBandwidthStats](#NetBandwidthStats) + * [NetBandwidthStatsByPeer](#NetBandwidthStatsByPeer) + * [NetBandwidthStatsByProtocol](#NetBandwidthStatsByProtocol) + * [NetConnect](#NetConnect) + * [NetConnectedness](#NetConnectedness) + * [NetDisconnect](#NetDisconnect) + * [NetFindPeer](#NetFindPeer) + * [NetPeers](#NetPeers) + * [NetPubsubScores](#NetPubsubScores) +* [Paych](#Paych) + * [PaychAllocateLane](#PaychAllocateLane) + * [PaychAvailableFunds](#PaychAvailableFunds) + * [PaychAvailableFundsByFromTo](#PaychAvailableFundsByFromTo) + * [PaychCollect](#PaychCollect) + * [PaychGet](#PaychGet) + * [PaychGetWaitReady](#PaychGetWaitReady) + * [PaychList](#PaychList) + * [PaychNewPayment](#PaychNewPayment) + * [PaychSettle](#PaychSettle) + * [PaychStatus](#PaychStatus) + * [PaychVoucherAdd](#PaychVoucherAdd) + * [PaychVoucherCheckSpendable](#PaychVoucherCheckSpendable) + * [PaychVoucherCheckValid](#PaychVoucherCheckValid) + * [PaychVoucherCreate](#PaychVoucherCreate) + * [PaychVoucherList](#PaychVoucherList) + * [PaychVoucherSubmit](#PaychVoucherSubmit) +* [State](#State) + * [StateAccountKey](#StateAccountKey) + * [StateAllMinerFaults](#StateAllMinerFaults) + * [StateCall](#StateCall) + * [StateChangedActors](#StateChangedActors) + * [StateCirculatingSupply](#StateCirculatingSupply) + * [StateCompute](#StateCompute) + * [StateDealProviderCollateralBounds](#StateDealProviderCollateralBounds) + * [StateGetActor](#StateGetActor) + * [StateGetReceipt](#StateGetReceipt) + * [StateListActors](#StateListActors) + * [StateListMessages](#StateListMessages) + * [StateListMiners](#StateListMiners) + * [StateLookupID](#StateLookupID) + * [StateMarketBalance](#StateMarketBalance) + * [StateMarketDeals](#StateMarketDeals) + * [StateMarketParticipants](#StateMarketParticipants) + * [StateMarketStorageDeal](#StateMarketStorageDeal) + * [StateMinerActiveSectors](#StateMinerActiveSectors) + * [StateMinerAvailableBalance](#StateMinerAvailableBalance) + * [StateMinerDeadlines](#StateMinerDeadlines) + * [StateMinerFaults](#StateMinerFaults) + * [StateMinerInfo](#StateMinerInfo) + * [StateMinerInitialPledgeCollateral](#StateMinerInitialPledgeCollateral) + * [StateMinerPartitions](#StateMinerPartitions) + * [StateMinerPower](#StateMinerPower) + * [StateMinerPreCommitDepositForPower](#StateMinerPreCommitDepositForPower) + * [StateMinerProvingDeadline](#StateMinerProvingDeadline) + * [StateMinerRecoveries](#StateMinerRecoveries) + * [StateMinerSectorCount](#StateMinerSectorCount) + * [StateMinerSectors](#StateMinerSectors) + * [StateMsgGasCost](#StateMsgGasCost) + * [StateNetworkName](#StateNetworkName) + * [StateReadState](#StateReadState) + * [StateReplay](#StateReplay) + * [StateSearchMsg](#StateSearchMsg) + * [StateSectorExpiration](#StateSectorExpiration) + * [StateSectorGetInfo](#StateSectorGetInfo) + * [StateSectorPartition](#StateSectorPartition) + * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) + * [StateVerifiedClientStatus](#StateVerifiedClientStatus) + * [StateWaitMsg](#StateWaitMsg) +* [Sync](#Sync) + * [SyncCheckBad](#SyncCheckBad) + * [SyncCheckpoint](#SyncCheckpoint) + * [SyncIncomingBlocks](#SyncIncomingBlocks) + * [SyncMarkBad](#SyncMarkBad) + * [SyncState](#SyncState) + * [SyncSubmitBlock](#SyncSubmitBlock) + * [SyncUnmarkBad](#SyncUnmarkBad) +* [Wallet](#Wallet) + * [WalletBalance](#WalletBalance) + * [WalletDefaultAddress](#WalletDefaultAddress) + * [WalletDelete](#WalletDelete) + * [WalletExport](#WalletExport) + * [WalletHas](#WalletHas) + * [WalletImport](#WalletImport) + * [WalletList](#WalletList) + * [WalletNew](#WalletNew) + * [WalletSetDefault](#WalletSetDefault) + * [WalletSign](#WalletSign) + * [WalletSignMessage](#WalletSignMessage) + * [WalletVerify](#WalletVerify) +## + + +### Closing + + +Perms: read + +Inputs: `null` + +Response: `{}` + +### Shutdown + + +Perms: admin + +Inputs: `null` + +Response: `{}` + +### Version + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Version": "string value", + "APIVersion": 3584, + "BlockDelay": 42 +} +``` + +## Auth + + +### AuthNew + + +Perms: admin + +Inputs: +```json +[ + null +] +``` + +Response: `"Ynl0ZSBhcnJheQ=="` + +### AuthVerify + + +Perms: read + +Inputs: +```json +[ + "string value" +] +``` + +Response: `null` + +## Beacon +The Beacon method group contains methods for interacting with the random beacon (DRAND) + + +### BeaconGetEntry +BeaconGetEntry returns the beacon entry for the given filecoin epoch. If +the entry has not yet been produced, the call will block until the entry +becomes available + + +Perms: read + +Inputs: +```json +[ + 10101 +] +``` + +Response: +```json +{ + "Round": 42, + "Data": "Ynl0ZSBhcnJheQ==" +} +``` + +## Chain +The Chain method group contains methods for interacting with the +blockchain, but that do not require any form of state computation. + + +### ChainExport +ChainExport returns a stream of bytes with CAR dump of chain data. +The exported chain data includes the header chain from the given tipset +back to genesis, the entire genesis state, and the most recent 'nroots' +state trees. +If oldmsgskip is set, messages from before the requested roots are also not included. + + +Perms: read + +Inputs: +```json +[ + 10101, + true, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"Ynl0ZSBhcnJheQ=="` + +### ChainGetBlock +ChainGetBlock returns the block specified by the given CID. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Miner": "t01234", + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "ElectionProof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "WinPoStProof": null, + "Parents": null, + "ParentWeight": "0", + "Height": 10101, + "ParentStateRoot": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ParentMessageReceipts": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Messages": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "BLSAggregate": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Timestamp": 42, + "BlockSig": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "ForkSignaling": 42, + "ParentBaseFee": "0" +} +``` + +### ChainGetBlockMessages +ChainGetBlockMessages returns messages stored in the specified block. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "BlsMessages": null, + "SecpkMessages": null, + "Cids": null +} +``` + +### ChainGetGenesis +ChainGetGenesis returns the genesis tipset. + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Cids": null, + "Blocks": null, + "Height": 0 +} +``` + +### ChainGetMessage +ChainGetMessage reads a message referenced by the specified CID from the +chain blockstore. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" +} +``` + +### ChainGetNode +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "string value" +] +``` + +Response: +```json +{ + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Obj": {} +} +``` + +### ChainGetParentMessages +ChainGetParentMessages returns messages stored in parent tipset of the +specified block. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `null` + +### ChainGetParentReceipts +ChainGetParentReceipts returns receipts for messages in parent tipset of +the specified block. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `null` + +### ChainGetPath +ChainGetPath returns a set of revert/apply operations needed to get from +one tipset to another, for example: +``` + to + ^ +from tAA + ^ ^ +tBA tAB + ^---*--^ + ^ + tRR +``` +Would return `[revert(tBA), apply(tAB), apply(tAA)]` + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### ChainGetRandomnessFromBeacon +ChainGetRandomnessFromBeacon is used to sample the beacon for randomness. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + 2, + 10101, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: `null` + +### ChainGetRandomnessFromTickets +ChainGetRandomnessFromTickets is used to sample the chain for randomness. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + 2, + 10101, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: `null` + +### ChainGetTipSet +ChainGetTipSet returns the tipset specified by the given TipSetKey. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Cids": null, + "Blocks": null, + "Height": 0 +} +``` + +### ChainGetTipSetByHeight +ChainGetTipSetByHeight looks back for a tipset at the specified epoch. +If there are no blocks at the specified epoch, a tipset at an earlier epoch +will be returned. + + +Perms: read + +Inputs: +```json +[ + 10101, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Cids": null, + "Blocks": null, + "Height": 0 +} +``` + +### ChainHasObj +ChainHasObj checks if a given CID exists in the chain blockstore. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `true` + +### ChainHead +ChainHead returns the current head of the chain. + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Cids": null, + "Blocks": null, + "Height": 0 +} +``` + +### ChainNotify +ChainNotify returns channel with chain head updates. +First message is guaranteed to be of len == 1, and type == 'current'. + + +Perms: read + +Inputs: `null` + +Response: `null` + +### ChainReadObj +ChainReadObj reads ipld nodes referenced by the specified CID from chain +blockstore and returns raw bytes. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `"Ynl0ZSBhcnJheQ=="` + +### ChainSetHead +ChainSetHead forcefully sets current chain head. Use with caution. + + +Perms: admin + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `{}` + +### ChainStatObj +ChainStatObj returns statistics about the graph referenced by 'obj'. +If 'base' is also specified, then the returned stat will be a diff +between the two objects. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Size": 42, + "Links": 42 +} +``` + +### ChainTipSetWeight +ChainTipSetWeight computes weight for the specified tipset. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +## Client +The Client methods all have to do with interacting with the storage and +retrieval markets as a client + + +### ClientCalcCommP +ClientCalcCommP calculates the CommP for a specified file + + +Perms: read + +Inputs: +```json +[ + "string value" +] +``` + +Response: +```json +{ + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Size": 1024 +} +``` + +### ClientDataTransferUpdates +There are not yet any comments for this method. + +Perms: write + +Inputs: `null` + +Response: +```json +{ + "TransferID": 3, + "Status": 1, + "BaseCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "IsInitiator": true, + "IsSender": true, + "Voucher": "string value", + "Message": "string value", + "OtherPeer": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "Transferred": 42 +} +``` + +### ClientDealSize +ClientDealSize calculates real deal data size + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "PayloadSize": 9, + "PieceSize": 1032 +} +``` + +### ClientFindData +ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer). + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + null +] +``` + +Response: `null` + +### ClientGenCar +ClientGenCar generates a CAR file for the specified file. + + +Perms: write + +Inputs: +```json +[ + { + "Path": "string value", + "IsCAR": true + }, + "string value" +] +``` + +Response: `{}` + +### ClientGetDealInfo +ClientGetDealInfo returns the latest information about a given deal. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "ProposalCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "State": 42, + "Message": "string value", + "Provider": "t01234", + "DataRef": { + "TransferType": "string value", + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceCid": null, + "PieceSize": 1024 + }, + "PieceCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Size": 42, + "PricePerEpoch": "0", + "Duration": 42, + "DealID": 5432, + "CreationTime": "0001-01-01T00:00:00Z" +} +``` + +### ClientGetDealUpdates +ClientGetDealUpdates returns the status of updated deals + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "ProposalCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "State": 42, + "Message": "string value", + "Provider": "t01234", + "DataRef": { + "TransferType": "string value", + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceCid": null, + "PieceSize": 1024 + }, + "PieceCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Size": 42, + "PricePerEpoch": "0", + "Duration": 42, + "DealID": 5432, + "CreationTime": "0001-01-01T00:00:00Z" +} +``` + +### ClientHasLocal +ClientHasLocal indicates whether a certain CID is locally stored. + + +Perms: write + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `true` + +### ClientImport +ClientImport imports file under the specified path into filestore. + + +Perms: admin + +Inputs: +```json +[ + { + "Path": "string value", + "IsCAR": true + } +] +``` + +Response: +```json +{ + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ImportID": 50 +} +``` + +### ClientListDataTransfers +ClientListTransfers returns the status of all ongoing transfers of data + + +Perms: write + +Inputs: `null` + +Response: `null` + +### ClientListDeals +ClientListDeals returns information about the deals made by the local client. + + +Perms: write + +Inputs: `null` + +Response: `null` + +### ClientListImports +ClientListImports lists imported files and their root CIDs + + +Perms: write + +Inputs: `null` + +Response: `null` + +### ClientMinerQueryOffer +ClientMinerQueryOffer returns a QueryOffer for the specific miner and file. + + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + null +] +``` + +Response: +```json +{ + "Err": "string value", + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Piece": null, + "Size": 42, + "MinPrice": "0", + "UnsealPrice": "0", + "PaymentInterval": 42, + "PaymentIntervalIncrease": 42, + "Miner": "t01234", + "MinerPeer": { + "Address": "t01234", + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "PieceCID": null + } +} +``` + +### ClientQueryAsk +ClientQueryAsk returns a signed StorageAsk from the specified miner. + + +Perms: read + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "t01234" +] +``` + +Response: +```json +{ + "Ask": { + "Price": "0", + "VerifiedPrice": "0", + "MinPieceSize": 1032, + "MaxPieceSize": 1032, + "Miner": "t01234", + "Timestamp": 10101, + "Expiry": 10101, + "SeqNo": 42 + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } +} +``` + +### ClientRemoveImport +ClientRemoveImport removes file import + + +Perms: admin + +Inputs: +```json +[ + 50 +] +``` + +Response: `{}` + +### ClientRetrieve +ClientRetrieve initiates the retrieval of a file, as specified in the order. + + +Perms: admin + +Inputs: +```json +[ + { + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Piece": null, + "Size": 42, + "Total": "0", + "UnsealPrice": "0", + "PaymentInterval": 42, + "PaymentIntervalIncrease": 42, + "Client": "t01234", + "Miner": "t01234", + "MinerPeer": { + "Address": "t01234", + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "PieceCID": null + } + }, + { + "Path": "string value", + "IsCAR": true + } +] +``` + +Response: `{}` + +### ClientRetrieveTryRestartInsufficientFunds +ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel +which are stuck due to insufficient funds + + +Perms: write + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `{}` + +### ClientRetrieveWithEvents +ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel +of status updates. + + +Perms: admin + +Inputs: +```json +[ + { + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Piece": null, + "Size": 42, + "Total": "0", + "UnsealPrice": "0", + "PaymentInterval": 42, + "PaymentIntervalIncrease": 42, + "Client": "t01234", + "Miner": "t01234", + "MinerPeer": { + "Address": "t01234", + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "PieceCID": null + } + }, + { + "Path": "string value", + "IsCAR": true + } +] +``` + +Response: +```json +{ + "Event": 5, + "Status": 0, + "BytesReceived": 42, + "FundsSpent": "0", + "Err": "string value" +} +``` + +### ClientStartDeal +ClientStartDeal proposes a deal with a miner. + + +Perms: admin + +Inputs: +```json +[ + { + "Data": { + "TransferType": "string value", + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceCid": null, + "PieceSize": 1024 + }, + "Wallet": "t01234", + "Miner": "t01234", + "EpochPrice": "0", + "MinBlocksDuration": 42, + "ProviderCollateral": "0", + "DealStartEpoch": 10101, + "FastRetrieval": true, + "VerifiedDeal": true + } +] +``` + +Response: `null` + +## Gas + + +### GasEstimateFeeCap +GasEstimateFeeCap estimates gas fee cap + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### GasEstimateGasLimit +GasEstimateGasLimit estimates gas used by the message and returns it. +It fails if message fails to execute. + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `9` + +### GasEstimateGasPremium +GasEstimateGasPremium estimates what gas price should be used for a +message to have high likelihood of inclusion in `nblocksincl` epochs. + + +Perms: read + +Inputs: +```json +[ + 42, + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### GasEstimateMessageGas +GasEstimateMessageGas estimates gas values for unset message gas fields + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + { + "MaxFee": "0" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" +} +``` + +## I + + +### ID + + +Perms: read + +Inputs: `null` + +Response: `"12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"` + +## Log + + +### LogList + + +Perms: write + +Inputs: `null` + +Response: `null` + +### LogSetLevel + + +Perms: write + +Inputs: +```json +[ + "string value", + "string value" +] +``` + +Response: `{}` + +## Market + + +### MarketEnsureAvailable +MarketFreeBalance + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "0" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +## Miner + + +### MinerCreateBlock +There are not yet any comments for this method. + +Perms: write + +Inputs: +```json +[ + { + "Miner": "t01234", + "Parents": [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "Eproof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconValues": null, + "Messages": null, + "Epoch": 10101, + "Timestamp": 42, + "WinningPoStProof": null + } +] +``` + +Response: +```json +{ + "Header": { + "Miner": "t01234", + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "ElectionProof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "WinPoStProof": null, + "Parents": null, + "ParentWeight": "0", + "Height": 10101, + "ParentStateRoot": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ParentMessageReceipts": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Messages": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "BLSAggregate": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Timestamp": 42, + "BlockSig": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "ForkSignaling": 42, + "ParentBaseFee": "0" + }, + "BlsMessages": null, + "SecpkMessages": null +} +``` + +### MinerGetBaseInfo +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "t01234", + 10101, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "MinerPower": "0", + "NetworkPower": "0", + "Sectors": null, + "WorkerKey": "t01234", + "SectorSize": 34359738368, + "PrevBeaconEntry": { + "Round": 42, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "HasMinPower": true +} +``` + +## Mpool +The Mpool methods are for interacting with the message pool. The message pool +manages all incoming and outgoing 'messages' going over the network. + + +### MpoolClear +MpoolClear clears pending messages from the mpool + + +Perms: write + +Inputs: +```json +[ + true +] +``` + +Response: `{}` + +### MpoolGetConfig +MpoolGetConfig returns (a copy of) the current mpool config + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "PriorityAddrs": null, + "SizeLimitHigh": 123, + "SizeLimitLow": 123, + "ReplaceByFeeRatio": 12.3, + "PruneCooldown": 60000000000, + "GasLimitOverestimation": 12.3 +} +``` + +### MpoolGetNonce +MpoolGetNonce gets next nonce for the specified sender. +Note that this method may not be atomic. Use MpoolPushMessage instead. + + +Perms: read + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `42` + +### MpoolPending +MpoolPending returns pending mempool messages. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### MpoolPush +MpoolPush pushes a signed message to mempool. + + +Perms: write + +Inputs: +```json +[ + { + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + } +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MpoolPushMessage +MpoolPushMessage atomically assigns a nonce, signs, and pushes a message +to mempool. +maxFee is only used when GasFeeCap/GasPremium fields aren't specified + +When maxFee is set to 0, MpoolPushMessage will guess appropriate fee +based on current chain conditions + + +Perms: sign + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + { + "MaxFee": "0" + } +] +``` + +Response: +```json +{ + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } +} +``` + +### MpoolSelect +MpoolSelect returns a list of pending messages for inclusion in the next block + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + 12.3 +] +``` + +Response: `null` + +### MpoolSetConfig +MpoolSetConfig sets the mpool config to (a copy of) the supplied config + + +Perms: write + +Inputs: +```json +[ + { + "PriorityAddrs": null, + "SizeLimitHigh": 123, + "SizeLimitLow": 123, + "ReplaceByFeeRatio": 12.3, + "PruneCooldown": 60000000000, + "GasLimitOverestimation": 12.3 + } +] +``` + +Response: `{}` + +### MpoolSub +There are not yet any comments for this method. + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Type": 0, + "Message": { + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + } +} +``` + +## Msig +The Msig methods are used to interact with multisig wallets on the +filecoin network + + +### MsigAddApprove +MsigAddApprove approves a previously proposed AddSigner message +It takes the following params: , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + 42, + "t01234", + "t01234", + true +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigAddCancel +MsigAddCancel cancels a previously proposed AddSigner message +It takes the following params: , , , +, + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + 42, + "t01234", + true +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigAddPropose +MsigAddPropose proposes adding a signer in the multisig +It takes the following params: , , +, + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "t01234", + true +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigApprove +MsigApprove approves a previously-proposed multisig message +It takes the following params: , , , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + 42, + "t01234", + "t01234", + "0", + "t01234", + 42, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigCancel +MsigCancel cancels a previously-proposed multisig message +It takes the following params: , , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + 42, + "t01234", + "0", + "t01234", + 42, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigCreate +MsigCreate creates a multisig wallet +It takes the following params: , , +, , + + +Perms: sign + +Inputs: +```json +[ + 42, + null, + 10101, + "0", + "t01234", + "0" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigGetAvailableBalance +MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### MsigGetVested +MsigGetVested returns the amount of FIL that vested in a multisig in a certain period. +It takes the following params: , , + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### MsigPropose +MsigPropose proposes a multisig message +It takes the following params: , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "0", + "t01234", + 42, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigSwapApprove +MsigSwapApprove approves a previously proposed SwapSigner +It takes the following params: , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + 42, + "t01234", + "t01234", + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigSwapCancel +MsigSwapCancel cancels a previously proposed SwapSigner message +It takes the following params: , , , +, + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + 42, + "t01234", + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigSwapPropose +MsigSwapPropose proposes swapping 2 signers in the multisig +It takes the following params: , , +, + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "t01234", + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +## Net + + +### NetAddrsListen + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Addrs": null, + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +} +``` + +### NetAgentVersion + + +Perms: read + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +] +``` + +Response: `"string value"` + +### NetAutoNatStatus + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Reachability": 1, + "PublicAddr": "string value" +} +``` + +### NetBandwidthStats + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "TotalIn": 9, + "TotalOut": 9, + "RateIn": 12.3, + "RateOut": 12.3 +} +``` + +### NetBandwidthStatsByPeer + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "12D3KooWSXmXLJmBR1M7i9RW9GQPNUhZSzXKzxDHWtAgNuJAbyEJ": { + "TotalIn": 174000, + "TotalOut": 12500, + "RateIn": 100, + "RateOut": 50 + } +} +``` + +### NetBandwidthStatsByProtocol + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "/fil/hello/1.0.0": { + "TotalIn": 174000, + "TotalOut": 12500, + "RateIn": 100, + "RateOut": 50 + } +} +``` + +### NetConnect + + +Perms: write + +Inputs: +```json +[ + { + "Addrs": null, + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" + } +] +``` + +Response: `{}` + +### NetConnectedness + + +Perms: read + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +] +``` + +Response: `1` + +### NetDisconnect + + +Perms: write + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +] +``` + +Response: `{}` + +### NetFindPeer + + +Perms: read + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +] +``` + +Response: +```json +{ + "Addrs": null, + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +} +``` + +### NetPeers + + +Perms: read + +Inputs: `null` + +Response: `null` + +### NetPubsubScores + + +Perms: read + +Inputs: `null` + +Response: `null` + +## Paych +The Paych methods are for interacting with and managing payment channels + + +### PaychAllocateLane +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `42` + +### PaychAvailableFunds +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "Channel": "\u003cempty\u003e", + "From": "t01234", + "To": "t01234", + "ConfirmedAmt": "0", + "PendingAmt": "0", + "PendingWaitSentinel": null, + "QueuedAmt": "0", + "VoucherReedeemedAmt": "0" +} +``` + +### PaychAvailableFundsByFromTo +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234" +] +``` + +Response: +```json +{ + "Channel": "\u003cempty\u003e", + "From": "t01234", + "To": "t01234", + "ConfirmedAmt": "0", + "PendingAmt": "0", + "PendingWaitSentinel": null, + "QueuedAmt": "0", + "VoucherReedeemedAmt": "0" +} +``` + +### PaychCollect +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### PaychGet +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "0" +] +``` + +Response: +```json +{ + "Channel": "t01234", + "WaitSentinel": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +} +``` + +### PaychGetWaitReady +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `"t01234"` + +### PaychList +There are not yet any comments for this method. + +Perms: read + +Inputs: `null` + +Response: `null` + +### PaychNewPayment +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + null +] +``` + +Response: +```json +{ + "Channel": "t01234", + "WaitSentinel": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Vouchers": null +} +``` + +### PaychSettle +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### PaychStatus +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "ControlAddr": "t01234", + "Direction": 1 +} +``` + +### PaychVoucherAdd +There are not yet any comments for this method. + +Perms: write + +Inputs: +```json +[ + "t01234", + { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + }, + "Ynl0ZSBhcnJheQ==", + "0" +] +``` + +Response: `"0"` + +### PaychVoucherCheckSpendable +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + }, + "Ynl0ZSBhcnJheQ==", + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: `true` + +### PaychVoucherCheckValid +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + } +] +``` + +Response: `{}` + +### PaychVoucherCreate +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + "0", + 42 +] +``` + +Response: +```json +{ + "Voucher": { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + }, + "Shortfall": "0" +} +``` + +### PaychVoucherList +There are not yet any comments for this method. + +Perms: write + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `null` + +### PaychVoucherSubmit +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + }, + "Ynl0ZSBhcnJheQ==", + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +## State +The State methods are used to query, inspect, and interact with chain state. +All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. +A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. + + +### StateAccountKey +StateAccountKey returns the public key address of the given ID address + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"t01234"` + +### StateAllMinerFaults +StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset + + +Perms: read + +Inputs: +```json +[ + 10101, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateCall +StateCall runs the given message and returns its result without any persisted changes. + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Msg": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ExecutionTrace": { + "Msg": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "Error": "string value", + "Duration": 60000000000, + "GasCharges": null, + "Subcalls": null + }, + "Error": "string value", + "Duration": 60000000000 +} +``` + +### StateChangedActors +StateChangedActors returns all the actors whose states change between the two given state CIDs +TODO: Should this take tipset keys instead? + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "t01236": { + "Code": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Head": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Nonce": 42, + "Balance": "0" + } +} +``` + +### StateCirculatingSupply +StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "FilVested": "0", + "FilMined": "0", + "FilBurnt": "0", + "FilLocked": "0", + "FilCirculating": "0" +} +``` + +### StateCompute +StateCompute is a flexible command that applies the given messages on the given tipset. +The messages are run as though the VM were at the provided height. + + +Perms: read + +Inputs: +```json +[ + 10101, + null, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Trace": null +} +``` + +### StateDealProviderCollateralBounds +StateDealProviderCollateralBounds returns the min and max collateral a storage provider +can issue. It takes the deal size and verified status as parameters. + + +Perms: read + +Inputs: +```json +[ + 1032, + true, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Min": "0", + "Max": "0" +} +``` + +### StateGetActor +StateGetActor returns the indicated actor's nonce and balance. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Code": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Head": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Nonce": 42, + "Balance": "0" +} +``` + +### StateGetReceipt +StateGetReceipt returns the message receipt for the given message + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 +} +``` + +### StateListActors +StateListActors returns the addresses of every actor in the state + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateListMessages +StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height. + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + 10101 +] +``` + +Response: `null` + +### StateListMiners +StateListMiners returns the addresses of every miner that has claimed power in the Power Actor + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateLookupID +StateLookupID retrieves the ID address of the given address + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"t01234"` + +### StateMarketBalance +StateMarketBalance looks up the Escrow and Locked balances of the given address in the Storage Market + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Escrow": "0", + "Locked": "0" +} +``` + +### StateMarketDeals +StateMarketDeals returns information about every deal in the Storage Market + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "t026363": { + "Proposal": { + "PieceCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceSize": 1032, + "VerifiedDeal": true, + "Client": "t01234", + "Provider": "t01234", + "Label": "string value", + "StartEpoch": 10101, + "EndEpoch": 10101, + "StoragePricePerEpoch": "0", + "ProviderCollateral": "0", + "ClientCollateral": "0" + }, + "State": { + "SectorStartEpoch": 10101, + "LastUpdatedEpoch": 10101, + "SlashEpoch": 10101 + } + } +} +``` + +### StateMarketParticipants +StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "t026363": { + "Escrow": "0", + "Locked": "0" + } +} +``` + +### StateMarketStorageDeal +StateMarketStorageDeal returns information about the indicated deal + + +Perms: read + +Inputs: +```json +[ + 5432, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Proposal": { + "PieceCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceSize": 1032, + "VerifiedDeal": true, + "Client": "t01234", + "Provider": "t01234", + "Label": "string value", + "StartEpoch": 10101, + "EndEpoch": 10101, + "StoragePricePerEpoch": "0", + "ProviderCollateral": "0", + "ClientCollateral": "0" + }, + "State": { + "SectorStartEpoch": 10101, + "LastUpdatedEpoch": 10101, + "SlashEpoch": 10101 + } +} +``` + +### StateMinerActiveSectors +StateMinerActiveSectors returns info about sectors that a given miner is actively proving. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateMinerAvailableBalance +StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateMinerDeadlines +StateMinerDeadlines returns all the proving deadlines for the given miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateMinerFaults +StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +[ + 5, + 1 +] +``` + +### StateMinerInfo +StateMinerInfo returns info about the indicated miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Owner": "t01234", + "Worker": "t01234", + "NewWorker": "t01234", + "ControlAddresses": null, + "WorkerChangeEpoch": 10101, + "PeerId": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "Multiaddrs": null, + "SealProofType": 3, + "SectorSize": 34359738368, + "WindowPoStPartitionSectors": 42 +} +``` + +### StateMinerInitialPledgeCollateral +StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector + + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "SealProof": 3, + "SectorNumber": 9, + "SealedCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "SealRandEpoch": 10101, + "DealIDs": null, + "Expiration": 10101, + "ReplaceCapacity": true, + "ReplaceSectorDeadline": 42, + "ReplaceSectorPartition": 42, + "ReplaceSectorNumber": 9 + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateMinerPartitions +StateMinerPartitions loads miner partitions for the specified miner/deadline + + +Perms: read + +Inputs: +```json +[ + "t01234", + 42, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateMinerPower +StateMinerPower returns the power of the indicated miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "MinerPower": { + "RawBytePower": "0", + "QualityAdjPower": "0" + }, + "TotalPower": { + "RawBytePower": "0", + "QualityAdjPower": "0" + } +} +``` + +### StateMinerPreCommitDepositForPower +StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector + + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "SealProof": 3, + "SectorNumber": 9, + "SealedCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "SealRandEpoch": 10101, + "DealIDs": null, + "Expiration": 10101, + "ReplaceCapacity": true, + "ReplaceSectorDeadline": 42, + "ReplaceSectorPartition": 42, + "ReplaceSectorNumber": 9 + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateMinerProvingDeadline +StateMinerProvingDeadline calculates the deadline at some epoch for a proving period +and returns the deadline-related calculations. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "CurrentEpoch": 10101, + "PeriodStart": 10101, + "Index": 42, + "Open": 10101, + "Close": 10101, + "Challenge": 10101, + "FaultCutoff": 10101, + "WPoStPeriodDeadlines": 42, + "WPoStProvingPeriod": 10101, + "WPoStChallengeWindow": 10101, + "WPoStChallengeLookback": 10101, + "FaultDeclarationCutoff": 10101 +} +``` + +### StateMinerRecoveries +StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +[ + 5, + 1 +] +``` + +### StateMinerSectorCount +StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Sectors": 42, + "Active": 42 +} +``` + +### StateMinerSectors +StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included. +If the filterOut boolean is set to true, any sectors in the filter are excluded. +If false, only those sectors in the filter are included. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + 0 + ], + true, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateMsgGasCost +StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" +} +``` + +### StateNetworkName +StateNetworkName returns the name of the network the node is synced to + + +Perms: read + +Inputs: `null` + +Response: `"lotus"` + +### StateReadState +StateReadState returns the indicated actor's state. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Balance": "0", + "State": {} +} +``` + +### StateReplay +StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Msg": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ExecutionTrace": { + "Msg": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "Error": "string value", + "Duration": 60000000000, + "GasCharges": null, + "Subcalls": null + }, + "Error": "string value", + "Duration": 60000000000 +} +``` + +### StateSearchMsg +StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Receipt": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ReturnDec": {}, + "TipSet": [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + "Height": 10101 +} +``` + +### StateSectorExpiration +StateSectorExpiration returns epoch at which given sector will expire + + +Perms: read + +Inputs: +```json +[ + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "OnTime": 10101, + "Early": 10101 +} +``` + +### StateSectorGetInfo +StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found +NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate +expiration epoch + + +Perms: read + +Inputs: +```json +[ + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "SectorNumber": 9, + "SealProof": 3, + "SealedCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "DealIDs": null, + "Activation": 10101, + "Expiration": 10101, + "DealWeight": "0", + "VerifiedDealWeight": "0", + "InitialPledge": "0", + "ExpectedDayReward": "0", + "ExpectedStoragePledge": "0" +} +``` + +### StateSectorPartition +StateSectorPartition finds deadline/partition with the specified sector + + +Perms: read + +Inputs: +```json +[ + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Deadline": 42, + "Partition": 42 +} +``` + +### StateSectorPreCommitInfo +StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector + + +Perms: read + +Inputs: +```json +[ + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Info": { + "SealProof": 3, + "SectorNumber": 9, + "SealedCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "SealRandEpoch": 10101, + "DealIDs": null, + "Expiration": 10101, + "ReplaceCapacity": true, + "ReplaceSectorDeadline": 42, + "ReplaceSectorPartition": 42, + "ReplaceSectorNumber": 9 + }, + "PreCommitDeposit": "0", + "PreCommitEpoch": 10101, + "DealWeight": "0", + "VerifiedDealWeight": "0" +} +``` + +### StateVerifiedClientStatus +StateVerifiedClientStatus returns the data cap for the given address. +Returns nil if there is no entry in the data cap table for the +address. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateWaitMsg +StateWaitMsg looks back in the chain for a message. If not found, it blocks until the +message arrives on chain, and gets to the indicated confidence depth. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + 42 +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Receipt": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ReturnDec": {}, + "TipSet": [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + "Height": 10101 +} +``` + +## Sync +The Sync method group contains methods for interacting with and +observing the lotus sync service. + + +### SyncCheckBad +SyncCheckBad checks if a block was marked as bad, and if it was, returns +the reason. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `"string value"` + +### SyncCheckpoint +SyncCheckpoint marks a blocks as checkpointed, meaning that it won't ever fork away from it. + + +Perms: admin + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `{}` + +### SyncIncomingBlocks +SyncIncomingBlocks returns a channel streaming incoming, potentially not +yet synced block headers. + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Miner": "t01234", + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "ElectionProof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "WinPoStProof": null, + "Parents": null, + "ParentWeight": "0", + "Height": 10101, + "ParentStateRoot": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ParentMessageReceipts": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Messages": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "BLSAggregate": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Timestamp": 42, + "BlockSig": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "ForkSignaling": 42, + "ParentBaseFee": "0" +} +``` + +### SyncMarkBad +SyncMarkBad marks a blocks as bad, meaning that it won't ever by synced. +Use with extreme caution. + + +Perms: admin + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `{}` + +### SyncState +SyncState returns the current status of the lotus sync system. + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "ActiveSyncs": null +} +``` + +### SyncSubmitBlock +SyncSubmitBlock can be used to submit a newly created block to the. +network through this node + + +Perms: write + +Inputs: +```json +[ + { + "Header": { + "Miner": "t01234", + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "ElectionProof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "WinPoStProof": null, + "Parents": null, + "ParentWeight": "0", + "Height": 10101, + "ParentStateRoot": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ParentMessageReceipts": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Messages": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "BLSAggregate": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Timestamp": 42, + "BlockSig": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "ForkSignaling": 42, + "ParentBaseFee": "0" + }, + "BlsMessages": null, + "SecpkMessages": null + } +] +``` + +Response: `{}` + +### SyncUnmarkBad +SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. + + +Perms: admin + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `{}` + +## Wallet + + +### WalletBalance +WalletBalance returns the balance of the given address at the current head of the chain. + + +Perms: read + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `"0"` + +### WalletDefaultAddress +WalletDefaultAddress returns the address marked as default in the wallet. + + +Perms: write + +Inputs: `null` + +Response: `"t01234"` + +### WalletDelete +WalletDelete deletes an address from the wallet. + + +Perms: write + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `{}` + +### WalletExport +WalletExport returns the private key of an address in the wallet. + + +Perms: admin + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "Type": "string value", + "PrivateKey": "Ynl0ZSBhcnJheQ==" +} +``` + +### WalletHas +WalletHas indicates whether the given address is in the wallet. + + +Perms: write + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `true` + +### WalletImport +WalletImport receives a KeyInfo, which includes a private key, and imports it into the wallet. + + +Perms: admin + +Inputs: +```json +[ + { + "Type": "string value", + "PrivateKey": "Ynl0ZSBhcnJheQ==" + } +] +``` + +Response: `"t01234"` + +### WalletList +WalletList lists all the addresses in the wallet. + + +Perms: write + +Inputs: `null` + +Response: `null` + +### WalletNew +WalletNew creates a new address in the wallet with the given sigType. + + +Perms: write + +Inputs: +```json +[ + 2 +] +``` + +Response: `"t01234"` + +### WalletSetDefault +WalletSetDefault marks the given address as as the default one. + + +Perms: admin + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `{}` + +### WalletSign +WalletSign signs the given bytes using the given address. + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" +} +``` + +### WalletSignMessage +WalletSignMessage signs the given message using the given address. + + +Perms: sign + +Inputs: +```json +[ + "t01234", + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + } +] +``` + +Response: +```json +{ + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } +} +``` + +### WalletVerify +WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid. +The address does not have to be in the wallet. + + +Perms: read + +Inputs: +```json +[ + "t01234", + "Ynl0ZSBhcnJheQ==", + { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } +] +``` + +Response: `true` + From dfd28ab0c44d7f032d6dff4e994b38491268ff0e Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:56:43 +0200 Subject: [PATCH 12/80] Fix links in READMEs. --- README.md | 4 ++-- tools/dockers/docker-examples/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6c1e23efa..fa432bf7d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- + Project Lotus Logo

@@ -18,7 +18,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more ## Building & Documentation -For instructions on how to build lotus from source, please visit [https://lotu.sh](https://lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). +For instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). ## Reporting a Vulnerability diff --git a/tools/dockers/docker-examples/README.md b/tools/dockers/docker-examples/README.md index 28553653c..3b8c34480 100644 --- a/tools/dockers/docker-examples/README.md +++ b/tools/dockers/docker-examples/README.md @@ -11,7 +11,7 @@ In this `docker-examples/` directory are community-contributed Docker and Docker - local node for a developer (`api-local-`) - hosted endpoint for apps / multiple developers (`api-hosted-`) - **For a local devnet or shared devnet** - - basic local devnet (also see [lotus docs on setting up a local devnet](https://lotu.sh/en+setup-local-dev-net)) + - basic local devnet (also see [lotus docs on setting up a local devnet](https://docs.filecoin.io/build/local-devnet/)) - shared devnet From a5e7fd8f5c0e8ed2c900bd23860e16d2336411f1 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 21 Sep 2020 18:43:57 +0200 Subject: [PATCH 13/80] Re-add docs-check step to circle --- .circleci/config.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8f149889..acd447f69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,6 +346,15 @@ jobs: - run: git --no-pager diff - run: git --no-pager diff --quiet + docs-check: + executor: golang + steps: + - install-deps + - prepare + - run: make docsgen + - run: git --no-pager diff + - run: git --no-pager diff --quiet + lint: &lint description: | Run golangci-lint. @@ -415,6 +424,7 @@ workflows: - mod-tidy-check - gofmt - cbor-gen-check + - docs-check - test: codecov-upload: true test-suite-name: full From ad3db0e83c5f6723573eec1464bc3bc48d29c001 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 21 Sep 2020 19:05:39 +0200 Subject: [PATCH 14/80] Move unclassified docs back to the docs root --- .../en/{unclassified => }/WIP-arch-complementary-notes.md | 0 documentation/en/{unclassified => }/block-validation.md | 0 documentation/en/{unclassified => }/create-miner.md | 0 documentation/en/{unclassified => }/dev-tools-pond-ui.md | 0 documentation/en/{unclassified => }/sealing-procs.md | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename documentation/en/{unclassified => }/WIP-arch-complementary-notes.md (100%) rename documentation/en/{unclassified => }/block-validation.md (100%) rename documentation/en/{unclassified => }/create-miner.md (100%) rename documentation/en/{unclassified => }/dev-tools-pond-ui.md (100%) rename documentation/en/{unclassified => }/sealing-procs.md (100%) diff --git a/documentation/en/unclassified/WIP-arch-complementary-notes.md b/documentation/en/WIP-arch-complementary-notes.md similarity index 100% rename from documentation/en/unclassified/WIP-arch-complementary-notes.md rename to documentation/en/WIP-arch-complementary-notes.md diff --git a/documentation/en/unclassified/block-validation.md b/documentation/en/block-validation.md similarity index 100% rename from documentation/en/unclassified/block-validation.md rename to documentation/en/block-validation.md diff --git a/documentation/en/unclassified/create-miner.md b/documentation/en/create-miner.md similarity index 100% rename from documentation/en/unclassified/create-miner.md rename to documentation/en/create-miner.md diff --git a/documentation/en/unclassified/dev-tools-pond-ui.md b/documentation/en/dev-tools-pond-ui.md similarity index 100% rename from documentation/en/unclassified/dev-tools-pond-ui.md rename to documentation/en/dev-tools-pond-ui.md diff --git a/documentation/en/unclassified/sealing-procs.md b/documentation/en/sealing-procs.md similarity index 100% rename from documentation/en/unclassified/sealing-procs.md rename to documentation/en/sealing-procs.md From 3c524ac0e0d806468e1e3a28437e94115089d477 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 23 Sep 2020 18:28:11 +0200 Subject: [PATCH 15/80] refactor: move nonce generation from mpool to wallet --- chain/messagepool/messagepool.go | 94 ------------- chain/messagesigner/messagesigner.go | 116 ++++++++++++++++ chain/messagesigner/messagesigner_test.go | 159 ++++++++++++++++++++++ node/builder.go | 3 + node/impl/full/mpool.go | 66 +++++---- 5 files changed, 310 insertions(+), 128 deletions(-) create mode 100644 chain/messagesigner/messagesigner.go create mode 100644 chain/messagesigner/messagesigner_test.go diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 96900925f..d54ea7164 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -75,8 +75,6 @@ var ( ErrRBFTooLowPremium = errors.New("replace by fee has too low GasPremium") ErrTooManyPendingMessages = errors.New("too many pending messages for actor") ErrNonceGap = errors.New("unfulfilled nonce gap") - - ErrTryAgain = errors.New("state inconsistency while pushing message; please try again") ) const ( @@ -795,98 +793,6 @@ func (mp *MessagePool) getStateBalance(addr address.Address, ts *types.TipSet) ( return act.Balance, nil } -func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address, cb func(address.Address, uint64) (*types.SignedMessage, error)) (*types.SignedMessage, error) { - // serialize push access to reduce lock contention - mp.addSema <- struct{}{} - defer func() { - <-mp.addSema - }() - - mp.curTsLk.Lock() - mp.lk.Lock() - - curTs := mp.curTs - - fromKey := addr - if fromKey.Protocol() == address.ID { - var err error - fromKey, err = mp.api.StateAccountKey(ctx, fromKey, mp.curTs) - if err != nil { - mp.lk.Unlock() - mp.curTsLk.Unlock() - return nil, xerrors.Errorf("resolving sender key: %w", err) - } - } - - nonce, err := mp.getNonceLocked(fromKey, mp.curTs) - if err != nil { - mp.lk.Unlock() - mp.curTsLk.Unlock() - return nil, xerrors.Errorf("get nonce locked failed: %w", err) - } - - // release the locks for signing - mp.lk.Unlock() - mp.curTsLk.Unlock() - - msg, err := cb(fromKey, nonce) - if err != nil { - return nil, err - } - - err = mp.checkMessage(msg) - if err != nil { - return nil, err - } - - msgb, err := msg.Serialize() - if err != nil { - return nil, err - } - - // reacquire the locks and check state for consistency - mp.curTsLk.Lock() - defer mp.curTsLk.Unlock() - - if mp.curTs != curTs { - return nil, ErrTryAgain - } - - mp.lk.Lock() - defer mp.lk.Unlock() - - nonce2, err := mp.getNonceLocked(fromKey, mp.curTs) - if err != nil { - return nil, xerrors.Errorf("get nonce locked failed: %w", err) - } - - if nonce2 != nonce { - return nil, ErrTryAgain - } - - publish, err := mp.verifyMsgBeforeAdd(msg, curTs, true) - if err != nil { - return nil, err - } - - if err := mp.checkBalance(msg, curTs); err != nil { - return nil, err - } - - if err := mp.addLocked(msg, false); err != nil { - return nil, xerrors.Errorf("add locked failed: %w", err) - } - if err := mp.addLocal(msg, msgb); err != nil { - log.Errorf("addLocal failed: %+v", err) - } - - if publish { - err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb) - } - - return msg, err -} - func (mp *MessagePool) Remove(from address.Address, nonce uint64, applied bool) { mp.lk.Lock() defer mp.lk.Unlock() diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go new file mode 100644 index 000000000..41b0edee9 --- /dev/null +++ b/chain/messagesigner/messagesigner.go @@ -0,0 +1,116 @@ +package messagesigner + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/wallet" + + "github.com/filecoin-project/lotus/chain/messagepool" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" +) + +const dsKeyActorNonce = "ActorNonce" + +type mpoolAPI interface { + GetNonce(address.Address) (uint64, error) +} + +// MessageSigner keeps track of nonces per address, and increments the nonce +// when signing a message +type MessageSigner struct { + wallet *wallet.Wallet + mpool mpoolAPI + ds datastore.Batching +} + +func NewMessageSigner(wallet *wallet.Wallet, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner { + return newMessageSigner(wallet, mpool, ds) +} + +func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner { + ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/")) + return &MessageSigner{ + wallet: wallet, + mpool: mpool, + ds: ds, + } +} + +// SignMessage increments the nonce for the message From address, and signs +// the message +func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) { + nonce, err := ms.nextNonce(msg.From) + if err != nil { + return nil, xerrors.Errorf("failed to create nonce: %w", err) + } + + msg.Nonce = nonce + sig, err := ms.wallet.Sign(ctx, msg.From, msg.Cid().Bytes()) + if err != nil { + return nil, xerrors.Errorf("failed to sign message: %w", err) + } + + return &types.SignedMessage{ + Message: *msg, + Signature: *sig, + }, nil +} + +// nextNonce increments the nonce. +// If there is no nonce in the datastore, gets the nonce from the message pool. +func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { + addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()}) + + // Get the nonce for this address from the datastore + nonceBytes, err := ms.ds.Get(addrNonceKey) + + var nonce uint64 + switch { + case xerrors.Is(err, datastore.ErrNotFound): + // If a nonce for this address hasn't yet been created in the + // datastore, check the mempool - nonces used to be created by + // the mempool so we need to support nodes that still have mempool + // nonces. Note that the mempool returns the actor state's nonce by + // default. + nonce, err = ms.mpool.GetNonce(addr) + if err != nil { + return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err) + } + + case err != nil: + return 0, xerrors.Errorf("failed to get nonce from datastore: %w", err) + + default: + // There is a nonce in the mempool, so unmarshall and increment it + maj, val, err := cbg.CborReadHeader(bytes.NewReader(nonceBytes)) + if err != nil { + return 0, xerrors.Errorf("failed to parse nonce from datastore: %w", err) + } + if maj != cbg.MajUnsignedInt { + return 0, xerrors.Errorf("bad cbor type parsing nonce from datastore") + } + + nonce = val + 1 + } + + // Write the nonce for this address to the datastore + buf := bytes.Buffer{} + _, err = buf.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, nonce)) + if err != nil { + return 0, xerrors.Errorf("failed to marshall nonce: %w", err) + } + err = ms.ds.Put(addrNonceKey, buf.Bytes()) + if err != nil { + return 0, xerrors.Errorf("failed to write nonce to datastore: %w", err) + } + + return nonce, nil +} diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go new file mode 100644 index 000000000..e52137892 --- /dev/null +++ b/chain/messagesigner/messagesigner_test.go @@ -0,0 +1,159 @@ +package messagesigner + +import ( + "context" + "sync" + "testing" + + "github.com/filecoin-project/lotus/chain/wallet" + + "github.com/filecoin-project/go-state-types/crypto" + "github.com/stretchr/testify/require" + + ds_sync "github.com/ipfs/go-datastore/sync" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-datastore" +) + +type mockMpool struct { + lk sync.RWMutex + nonces map[address.Address]uint64 +} + +func newMockMpool() *mockMpool { + return &mockMpool{nonces: make(map[address.Address]uint64)} +} + +func (mp *mockMpool) setNonce(addr address.Address, nonce uint64) { + mp.lk.Lock() + defer mp.lk.Unlock() + + mp.nonces[addr] = nonce +} + +func (mp *mockMpool) GetNonce(addr address.Address) (uint64, error) { + mp.lk.RLock() + defer mp.lk.RUnlock() + + return mp.nonces[addr], nil +} + +func TestMessageSignerSignMessage(t *testing.T) { + ctx := context.Background() + + w, _ := wallet.NewWallet(wallet.NewMemKeyStore()) + from1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + require.NoError(t, err) + from2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + require.NoError(t, err) + to1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + require.NoError(t, err) + to2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + require.NoError(t, err) + + type msgSpec struct { + msg *types.Message + mpoolNonce [1]uint64 + expNonce uint64 + } + tests := []struct { + name string + msgs []msgSpec + }{{ + // No nonce yet in datastore + name: "no nonce yet", + msgs: []msgSpec{{ + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 0, + }}, + }, { + // Get nonce value of zero from mpool + name: "mpool nonce zero", + msgs: []msgSpec{{ + msg: &types.Message{ + To: to1, + From: from1, + }, + mpoolNonce: [1]uint64{0}, + expNonce: 0, + }}, + }, { + // Get non-zero nonce value from mpool + name: "mpool nonce set", + msgs: []msgSpec{{ + msg: &types.Message{ + To: to1, + From: from1, + }, + mpoolNonce: [1]uint64{5}, + expNonce: 5, + }, { + msg: &types.Message{ + To: to1, + From: from1, + }, + // Should ignore mpool nonce because after the first message nonce + // will come from the datastore + mpoolNonce: [1]uint64{10}, + expNonce: 6, + }}, + }, { + // Nonce should increment independently for each address + name: "nonce increments per address", + msgs: []msgSpec{{ + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 0, + }, { + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 1, + }, { + msg: &types.Message{ + To: to2, + From: from2, + }, + mpoolNonce: [1]uint64{5}, + expNonce: 5, + }, { + msg: &types.Message{ + To: to2, + From: from2, + }, + expNonce: 6, + }, { + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 2, + }}, + }} + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + mpool := newMockMpool() + ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) + ms := newMessageSigner(w, mpool, ds) + + for _, m := range tt.msgs { + if len(m.mpoolNonce) == 1 { + mpool.setNonce(m.msg.From, m.mpoolNonce[0]) + } + smsg, err := ms.SignMessage(ctx, m.msg) + require.NoError(t, err) + require.Equal(t, m.expNonce, smsg.Message.Nonce) + } + }) + } +} diff --git a/node/builder.go b/node/builder.go index c37a5db58..c49789a6a 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,6 +6,8 @@ import ( "os" "time" + "github.com/filecoin-project/lotus/chain/messagesigner" + logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -259,6 +261,7 @@ func Online() Option { Override(new(*store.ChainStore), modules.ChainStore), Override(new(*stmgr.StateManager), stmgr.NewStateManager), Override(new(*wallet.Wallet), wallet.NewWallet), + Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 6acb17990..003260496 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -4,14 +4,14 @@ import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/chain/messagesigner" + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -22,9 +22,7 @@ type MpoolAPI struct { WalletAPI GasAPI - Chain *store.ChainStore - - Mpool *messagepool.MessagePool + MessageSigner *messagesigner.MessageSigner PushLocks *dtypes.MpoolLocker } @@ -114,12 +112,14 @@ func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (ci } func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) { + cp := *msg + msg = &cp inMsg := *msg + fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msg.From, nil) + if err != nil { + return nil, xerrors.Errorf("getting key address: %w", err) + } { - fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msg.From, nil) - if err != nil { - return nil, xerrors.Errorf("getting key address: %w", err) - } done, err := a.PushLocks.TakeLock(ctx, fromA) if err != nil { return nil, xerrors.Errorf("taking lock: %w", err) @@ -131,7 +131,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe return nil, xerrors.Errorf("MpoolPushMessage expects message nonce to be 0, was %d", msg.Nonce) } - msg, err := a.GasAPI.GasEstimateMessageGas(ctx, msg, spec, types.EmptyTSK) + msg, err = a.GasAPI.GasEstimateMessageGas(ctx, msg, spec, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("GasEstimateMessageGas error: %w", err) } @@ -143,33 +143,31 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe inJson, outJson) } - sign := func(from address.Address, nonce uint64) (*types.SignedMessage, error) { - msg.Nonce = nonce - if msg.From.Protocol() == address.ID { - log.Warnf("Push from ID address (%s), adjusting to %s", msg.From, from) - msg.From = from - } - - b, err := a.WalletBalance(ctx, msg.From) - if err != nil { - return nil, xerrors.Errorf("mpool push: getting origin balance: %w", err) - } - - if b.LessThan(msg.Value) { - return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value) - } - - return a.WalletSignMessage(ctx, from, msg) + if msg.From.Protocol() == address.ID { + log.Warnf("Push from ID address (%s), adjusting to %s", msg.From, fromA) + msg.From = fromA } - var m *types.SignedMessage -again: - m, err = a.Mpool.PushWithNonce(ctx, msg.From, sign) - if err == messagepool.ErrTryAgain { - log.Debugf("temporary failure while pushing message: %s; retrying", err) - goto again + b, err := a.WalletBalance(ctx, msg.From) + if err != nil { + return nil, xerrors.Errorf("mpool push: getting origin balance: %w", err) } - return m, err + + if b.LessThan(msg.Value) { + return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value) + } + + smsg, err := a.MessageSigner.SignMessage(ctx, msg) + if err != nil { + return nil, xerrors.Errorf("mpool push: failed to sign message: %w", err) + } + + _, err = a.Mpool.Push(smsg) + if err != nil { + return nil, xerrors.Errorf("mpool push: failed to push message: %w", err) + } + + return smsg, err } func (a *MpoolAPI) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) { From 7eb9bec13f6d351e192b0e8319ccf1bf0c3c077e Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 18 Sep 2020 18:03:59 +0200 Subject: [PATCH 16/80] feat: dont recompute post on submit redux --- storage/wdpost_changehandler.go | 533 ++++++++++++ storage/wdpost_changehandler_test.go | 1173 ++++++++++++++++++++++++++ storage/wdpost_nextdl_test.go | 38 + storage/wdpost_run.go | 193 +++-- storage/wdpost_run_test.go | 6 +- storage/wdpost_sched.go | 142 +--- 6 files changed, 1913 insertions(+), 172 deletions(-) create mode 100644 storage/wdpost_changehandler.go create mode 100644 storage/wdpost_changehandler_test.go create mode 100644 storage/wdpost_nextdl_test.go diff --git a/storage/wdpost_changehandler.go b/storage/wdpost_changehandler.go new file mode 100644 index 000000000..e65b7a7fc --- /dev/null +++ b/storage/wdpost_changehandler.go @@ -0,0 +1,533 @@ +package storage + +import ( + "context" + "sync" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/lotus/chain/types" +) + +const SubmitConfidence = 4 + +type CompleteGeneratePoSTCb func(posts []miner.SubmitWindowedPoStParams, err error) +type CompleteSubmitPoSTCb func(err error) + +type changeHandlerAPI interface { + StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) + startGeneratePoST(ctx context.Context, ts *types.TipSet, deadline *dline.Info, onComplete CompleteGeneratePoSTCb) context.CancelFunc + startSubmitPoST(ctx context.Context, ts *types.TipSet, deadline *dline.Info, posts []miner.SubmitWindowedPoStParams, onComplete CompleteSubmitPoSTCb) context.CancelFunc + onAbort(ts *types.TipSet, deadline *dline.Info) + failPost(err error, ts *types.TipSet, deadline *dline.Info) +} + +type changeHandler struct { + api changeHandlerAPI + actor address.Address + proveHdlr *proveHandler + submitHdlr *submitHandler +} + +func newChangeHandler(api changeHandlerAPI, actor address.Address) *changeHandler { + posts := newPostsCache() + p := newProver(api, posts) + s := newSubmitter(api, posts) + return &changeHandler{api: api, actor: actor, proveHdlr: p, submitHdlr: s} +} + +func (ch *changeHandler) start() { + go ch.proveHdlr.run() + go ch.submitHdlr.run() +} + +func (ch *changeHandler) update(ctx context.Context, revert *types.TipSet, advance *types.TipSet) error { + // Get the current deadline period + di, err := ch.api.StateMinerProvingDeadline(ctx, ch.actor, advance.Key()) + if err != nil { + return err + } + + if !di.PeriodStarted() { + return nil // not proving anything yet + } + + hc := &headChange{ + ctx: ctx, + revert: revert, + advance: advance, + di: di, + } + + select { + case ch.proveHdlr.hcs <- hc: + case <-ch.proveHdlr.shutdownCtx.Done(): + case <-ctx.Done(): + } + + select { + case ch.submitHdlr.hcs <- hc: + case <-ch.submitHdlr.shutdownCtx.Done(): + case <-ctx.Done(): + } + + return nil +} + +func (ch *changeHandler) shutdown() { + ch.proveHdlr.shutdown() + ch.submitHdlr.shutdown() +} + +func (ch *changeHandler) currentTSDI() (*types.TipSet, *dline.Info) { + return ch.submitHdlr.currentTSDI() +} + +// postsCache keeps a cache of PoSTs for each proving window +type postsCache struct { + added chan *postInfo + lk sync.RWMutex + cache map[abi.ChainEpoch][]miner.SubmitWindowedPoStParams +} + +func newPostsCache() *postsCache { + return &postsCache{ + added: make(chan *postInfo, 16), + cache: make(map[abi.ChainEpoch][]miner.SubmitWindowedPoStParams), + } +} + +func (c *postsCache) add(di *dline.Info, posts []miner.SubmitWindowedPoStParams) { + c.lk.Lock() + defer c.lk.Unlock() + + // TODO: clear cache entries older than chain finality + c.cache[di.Open] = posts + + c.added <- &postInfo{ + di: di, + posts: posts, + } +} + +func (c *postsCache) get(di *dline.Info) ([]miner.SubmitWindowedPoStParams, bool) { + c.lk.RLock() + defer c.lk.RUnlock() + + posts, ok := c.cache[di.Open] + return posts, ok +} + +type headChange struct { + ctx context.Context + revert *types.TipSet + advance *types.TipSet + di *dline.Info +} + +type currentPost struct { + di *dline.Info + abort context.CancelFunc +} + +type postResult struct { + ts *types.TipSet + currPost *currentPost + posts []miner.SubmitWindowedPoStParams + err error +} + +// proveHandler generates proofs +type proveHandler struct { + api changeHandlerAPI + posts *postsCache + + postResults chan *postResult + hcs chan *headChange + + current *currentPost + + shutdownCtx context.Context + shutdown context.CancelFunc + + // Used for testing + processedHeadChanges chan *headChange + processedPostResults chan *postResult +} + +func newProver( + api changeHandlerAPI, + posts *postsCache, +) *proveHandler { + ctx, cancel := context.WithCancel(context.Background()) + return &proveHandler{ + api: api, + posts: posts, + postResults: make(chan *postResult), + hcs: make(chan *headChange), + shutdownCtx: ctx, + shutdown: cancel, + } +} + +func (p *proveHandler) run() { + // Abort proving on shutdown + defer func() { + if p.current != nil { + p.current.abort() + } + }() + + for p.shutdownCtx.Err() == nil { + select { + case <-p.shutdownCtx.Done(): + return + + case hc := <-p.hcs: + // Head changed + p.processHeadChange(hc.ctx, hc.advance, hc.di) + if p.processedHeadChanges != nil { + p.processedHeadChanges <- hc + } + + case res := <-p.postResults: + // Proof generation complete + p.processPostResult(res) + if p.processedPostResults != nil { + p.processedPostResults <- res + } + } + } +} + +func (p *proveHandler) processHeadChange(ctx context.Context, newTS *types.TipSet, di *dline.Info) { + // If the post window has expired, abort the current proof + if p.current != nil && newTS.Height() >= p.current.di.Close { + // Cancel the context on the current proof + p.current.abort() + + // Clear out the reference to the proof so that we can immediately + // start generating a new proof, without having to worry about state + // getting clobbered when the abort completes + p.current = nil + } + + // Only generate one proof at a time + if p.current != nil { + return + } + + // If the proof for the current post window has been generated, check the + // next post window + _, complete := p.posts.get(di) + for complete { + di = nextDeadline(di) + _, complete = p.posts.get(di) + } + + // Check if the chain is above the Challenge height for the post window + if newTS.Height() < di.Challenge { + return + } + + p.current = ¤tPost{di: di} + curr := p.current + p.current.abort = p.api.startGeneratePoST(ctx, newTS, di, func(posts []miner.SubmitWindowedPoStParams, err error) { + p.postResults <- &postResult{ts: newTS, currPost: curr, posts: posts, err: err} + }) +} + +func (p *proveHandler) processPostResult(res *postResult) { + di := res.currPost.di + if res.err != nil { + // Proving failed so inform the API + p.api.failPost(res.err, res.ts, di) + log.Warnf("Aborted window post Proving (Deadline: %+v)", di) + p.api.onAbort(res.ts, di) + + // Check if the current post has already been aborted + if p.current == res.currPost { + // If the current post was not already aborted, setting it to nil + // marks it as complete so that a new post can be started + p.current = nil + } + return + } + + // Completed processing this proving window + p.current = nil + + // Add the proofs to the cache + p.posts.add(di, res.posts) +} + +type submitResult struct { + pw *postWindow + err error +} + +type SubmitState string + +const ( + SubmitStateStart SubmitState = "SubmitStateStart" + SubmitStateSubmitting SubmitState = "SubmitStateSubmitting" + SubmitStateComplete SubmitState = "SubmitStateComplete" +) + +type postWindow struct { + ts *types.TipSet + di *dline.Info + submitState SubmitState + abort context.CancelFunc +} + +type postInfo struct { + di *dline.Info + posts []miner.SubmitWindowedPoStParams +} + +// submitHandler submits proofs on-chain +type submitHandler struct { + api changeHandlerAPI + posts *postsCache + + submitResults chan *submitResult + hcs chan *headChange + + postWindows map[abi.ChainEpoch]*postWindow + getPostWindowReqs chan *getPWReq + + shutdownCtx context.Context + shutdown context.CancelFunc + + currentCtx context.Context + currentTS *types.TipSet + currentDI *dline.Info + getTSDIReq chan chan *tsdi + + // Used for testing + processedHeadChanges chan *headChange + processedSubmitResults chan *submitResult + processedPostReady chan *postInfo +} + +func newSubmitter( + api changeHandlerAPI, + posts *postsCache, +) *submitHandler { + ctx, cancel := context.WithCancel(context.Background()) + return &submitHandler{ + api: api, + posts: posts, + submitResults: make(chan *submitResult), + hcs: make(chan *headChange), + postWindows: make(map[abi.ChainEpoch]*postWindow), + getPostWindowReqs: make(chan *getPWReq), + getTSDIReq: make(chan chan *tsdi), + shutdownCtx: ctx, + shutdown: cancel, + } +} + +func (s *submitHandler) run() { + // On shutdown, abort in-progress submits + defer func() { + for _, pw := range s.postWindows { + if pw.abort != nil { + pw.abort() + } + } + }() + + for s.shutdownCtx.Err() == nil { + select { + case <-s.shutdownCtx.Done(): + return + + case hc := <-s.hcs: + // Head change + s.processHeadChange(hc.ctx, hc.revert, hc.advance, hc.di) + if s.processedHeadChanges != nil { + s.processedHeadChanges <- hc + } + + case pi := <-s.posts.added: + // Proof generated + s.processPostReady(pi) + if s.processedPostReady != nil { + s.processedPostReady <- pi + } + + case res := <-s.submitResults: + // Submit complete + s.processSubmitResult(res) + if s.processedSubmitResults != nil { + s.processedSubmitResults <- res + } + + case pwreq := <-s.getPostWindowReqs: + // used by getPostWindow() to sync with run loop + pwreq.out <- s.postWindows[pwreq.di.Open] + + case out := <-s.getTSDIReq: + // used by currentTSDI() to sync with run loop + out <- &tsdi{ts: s.currentTS, di: s.currentDI} + } + } +} + +// processHeadChange is called when the chain head changes +func (s *submitHandler) processHeadChange(ctx context.Context, revert *types.TipSet, advance *types.TipSet, di *dline.Info) { + s.currentCtx = ctx + s.currentTS = advance + s.currentDI = di + + // Start tracking the current post window if we're not already + // TODO: clear post windows older than chain finality + if _, ok := s.postWindows[di.Open]; !ok { + s.postWindows[di.Open] = &postWindow{ + di: di, + ts: advance, + submitState: SubmitStateStart, + } + } + + // Apply the change to all post windows + for _, pw := range s.postWindows { + s.processHeadChangeForPW(ctx, revert, advance, pw) + } +} + +func (s *submitHandler) processHeadChangeForPW(ctx context.Context, revert *types.TipSet, advance *types.TipSet, pw *postWindow) { + revertedToPrevDL := revert != nil && revert.Height() < pw.di.Open + expired := advance.Height() >= pw.di.Close + + // If the chain was reverted back to the previous deadline, or if the post + // window has expired, abort submit + if pw.submitState == SubmitStateSubmitting && (revertedToPrevDL || expired) { + // Replace the aborted postWindow with a new one so that we can + // submit again at any time without the state getting clobbered + // when the abort completes + abort := pw.abort + if abort != nil { + pw = &postWindow{ + di: pw.di, + ts: advance, + submitState: SubmitStateStart, + } + s.postWindows[pw.di.Open] = pw + + // Abort the current submit + abort() + } + } else if pw.submitState == SubmitStateComplete && revertedToPrevDL { + // If submit for this deadline has completed, but the chain was + // reverted back to the previous deadline, reset the submit state to the + // starting state, so that it can be resubmitted + pw.submitState = SubmitStateStart + } + + // Submit the proof to chain if the proof has been generated and the chain + // height is above confidence + s.submitIfReady(ctx, advance, pw) +} + +// processPostReady is called when a proof generation completes +func (s *submitHandler) processPostReady(pi *postInfo) { + pw, ok := s.postWindows[pi.di.Open] + if ok { + s.submitIfReady(s.currentCtx, s.currentTS, pw) + } +} + +// submitIfReady submits a proof if the chain is high enough and the proof +// has been generated for this deadline +func (s *submitHandler) submitIfReady(ctx context.Context, advance *types.TipSet, pw *postWindow) { + // If the window has expired, there's nothing more to do. + if advance.Height() >= pw.di.Close { + return + } + + // Check if we're already submitting, or already completed submit + if pw.submitState != SubmitStateStart { + return + } + + // Check if we've reached the confidence height to submit + if advance.Height() < pw.di.Open+SubmitConfidence { + return + } + + // Check if the proofs have been generated for this deadline + posts, ok := s.posts.get(pw.di) + if !ok { + return + } + + // If there was nothing to prove, move straight to the complete state + if len(posts) == 0 { + pw.submitState = SubmitStateComplete + return + } + + // Start submitting post + pw.submitState = SubmitStateSubmitting + pw.abort = s.api.startSubmitPoST(ctx, advance, pw.di, posts, func(err error) { + s.submitResults <- &submitResult{pw: pw, err: err} + }) +} + +// processSubmitResult is called with the response to a submit +func (s *submitHandler) processSubmitResult(res *submitResult) { + if res.err != nil { + // Submit failed so inform the API and go back to the start state + s.api.failPost(res.err, res.pw.ts, res.pw.di) + log.Warnf("Aborted window post Submitting (Deadline: %+v)", res.pw.di) + s.api.onAbort(res.pw.ts, res.pw.di) + + res.pw.submitState = SubmitStateStart + return + } + + // Submit succeeded so move to complete state + res.pw.submitState = SubmitStateComplete +} + +type tsdi struct { + ts *types.TipSet + di *dline.Info +} + +func (s *submitHandler) currentTSDI() (*types.TipSet, *dline.Info) { + out := make(chan *tsdi) + s.getTSDIReq <- out + res := <-out + return res.ts, res.di +} + +type getPWReq struct { + di *dline.Info + out chan *postWindow +} + +func (s *submitHandler) getPostWindow(di *dline.Info) *postWindow { + out := make(chan *postWindow) + s.getPostWindowReqs <- &getPWReq{di: di, out: out} + return <-out +} + +// nextDeadline gets deadline info for the subsequent deadline +func nextDeadline(currentDeadline *dline.Info) *dline.Info { + periodStart := currentDeadline.PeriodStart + newDeadline := currentDeadline.Index + 1 + if newDeadline == miner.WPoStPeriodDeadlines { + newDeadline = 0 + periodStart = periodStart + miner.WPoStProvingPeriod + } + + return miner.NewDeadlineInfo(periodStart, newDeadline, currentDeadline.CurrentEpoch) +} diff --git a/storage/wdpost_changehandler_test.go b/storage/wdpost_changehandler_test.go new file mode 100644 index 000000000..d2a4779e6 --- /dev/null +++ b/storage/wdpost_changehandler_test.go @@ -0,0 +1,1173 @@ +package storage + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + tutils "github.com/filecoin-project/specs-actors/support/testing" + + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +var dummyCid cid.Cid + +func init() { + dummyCid, _ = cid.Parse("bafkqaaa") +} + +type proveRes struct { + posts []miner.SubmitWindowedPoStParams + err error +} + +type postStatus string + +const ( + postStatusStart postStatus = "postStatusStart" + postStatusProving postStatus = "postStatusProving" + postStatusComplete postStatus = "postStatusComplete" +) + +type mockAPI struct { + ch *changeHandler + deadline *dline.Info + proveResult chan *proveRes + submitResult chan error + onStateChange chan struct{} + + tsLock sync.RWMutex + ts map[types.TipSetKey]*types.TipSet + + abortCalledLock sync.RWMutex + abortCalled bool + + statesLk sync.RWMutex + postStates map[abi.ChainEpoch]postStatus +} + +func newMockAPI() *mockAPI { + return &mockAPI{ + proveResult: make(chan *proveRes), + onStateChange: make(chan struct{}), + submitResult: make(chan error), + postStates: make(map[abi.ChainEpoch]postStatus), + ts: make(map[types.TipSetKey]*types.TipSet), + } +} + +func (m *mockAPI) makeTs(t *testing.T, h abi.ChainEpoch) *types.TipSet { + m.tsLock.Lock() + defer m.tsLock.Unlock() + + ts := makeTs(t, h) + m.ts[ts.Key()] = ts + return ts +} + +func (m *mockAPI) setDeadline(di *dline.Info) { + m.tsLock.Lock() + defer m.tsLock.Unlock() + + m.deadline = di +} + +func (m *mockAPI) getDeadline(currentEpoch abi.ChainEpoch) *dline.Info { + close := miner.WPoStChallengeWindow - 1 + dlIdx := uint64(0) + for close < currentEpoch { + close += miner.WPoStChallengeWindow + dlIdx++ + } + return miner.NewDeadlineInfo(0, dlIdx, currentEpoch) +} + +func (m *mockAPI) StateMinerProvingDeadline(ctx context.Context, address address.Address, key types.TipSetKey) (*dline.Info, error) { + m.tsLock.RLock() + defer m.tsLock.RUnlock() + + ts, ok := m.ts[key] + if !ok { + panic(fmt.Sprintf("unexpected tipset key %s", key)) + } + + if m.deadline != nil { + m.deadline.CurrentEpoch = ts.Height() + return m.deadline, nil + } + + return m.getDeadline(ts.Height()), nil +} + +func (m *mockAPI) startGeneratePoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + completeGeneratePoST CompleteGeneratePoSTCb, +) context.CancelFunc { + ctx, cancel := context.WithCancel(ctx) + + m.statesLk.Lock() + defer m.statesLk.Unlock() + m.postStates[deadline.Open] = postStatusProving + + go func() { + defer cancel() + + select { + case psRes := <-m.proveResult: + m.statesLk.Lock() + { + if psRes.err == nil { + m.postStates[deadline.Open] = postStatusComplete + } else { + m.postStates[deadline.Open] = postStatusStart + } + } + m.statesLk.Unlock() + completeGeneratePoST(psRes.posts, psRes.err) + case <-ctx.Done(): + completeGeneratePoST(nil, ctx.Err()) + } + }() + + return cancel +} + +func (m *mockAPI) getPostStatus(di *dline.Info) postStatus { + m.statesLk.RLock() + defer m.statesLk.RUnlock() + + status, ok := m.postStates[di.Open] + if ok { + return status + } + return postStatusStart +} + +func (m *mockAPI) startSubmitPoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + posts []miner.SubmitWindowedPoStParams, + completeSubmitPoST CompleteSubmitPoSTCb, +) context.CancelFunc { + ctx, cancel := context.WithCancel(ctx) + + go func() { + defer cancel() + + select { + case err := <-m.submitResult: + completeSubmitPoST(err) + case <-ctx.Done(): + completeSubmitPoST(ctx.Err()) + } + }() + + return cancel +} + +func (m *mockAPI) onAbort(ts *types.TipSet, deadline *dline.Info) { + m.abortCalledLock.Lock() + defer m.abortCalledLock.Unlock() + m.abortCalled = true +} + +func (m *mockAPI) wasAbortCalled() bool { + m.abortCalledLock.RLock() + defer m.abortCalledLock.RUnlock() + return m.abortCalled +} + +func (m *mockAPI) failPost(err error, ts *types.TipSet, deadline *dline.Info) { +} + +func (m *mockAPI) setChangeHandler(ch *changeHandler) { + m.ch = ch +} + +// TestChangeHandlerBasic verifies we can generate a proof and submit it +func TestChangeHandlerBasic(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send a response to the submit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerFromProvingToSubmittingNoHeadChange tests that when the +// chain is already advanced past the confidence interval, we should move from +// proving to submitting without a head change in between. +func TestChangeHandlerFromProvingToSubmittingNoHeadChange(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Monitor submit handler's processing of incoming postInfo + s.ch.submitHdlr.processedPostReady = make(chan *postInfo) + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Trigger a head change that advances the chain beyond the submit + // confidence + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should be no change to state yet + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + di = mock.getDeadline(currentEpoch) + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Should move directly to submitting state with no further head changes + <-s.ch.submitHdlr.processedPostReady + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) +} + +// TestChangeHandlerFromProvingEmptyProofsToComplete tests that when there are no +// proofs generated we should not submit anything to chain but submit state +// should move to completed +func TestChangeHandlerFromProvingEmptyProofsToComplete(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Monitor submit handler's processing of incoming postInfo + s.ch.submitHdlr.processedPostReady = make(chan *postInfo) + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Trigger a head change that advances the chain beyond the submit + // confidence + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should be no change to state yet + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs with an empty proofs array + posts := []miner.SubmitWindowedPoStParams{} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + di = mock.getDeadline(currentEpoch) + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Should move directly to submitting complete state + <-s.ch.submitHdlr.processedPostReady + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerDontStartUntilProvingPeriod tests that the handler +// ignores updates until the proving period has been reached. +func TestChangeHandlerDontStartUntilProvingPeriod(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + periodStart := miner.WPoStProvingPeriod + dlIdx := uint64(1) + currentEpoch := abi.ChainEpoch(10) + di := miner.NewDeadlineInfo(periodStart, dlIdx, currentEpoch) + mock.setDeadline(di) + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + go triggerHeadAdvance(t, s, currentEpoch) + + // Nothing should happen because the proving period has not started + select { + case <-s.ch.proveHdlr.processedHeadChanges: + require.Fail(t, "unexpected prove change") + case <-s.ch.submitHdlr.processedHeadChanges: + require.Fail(t, "unexpected submit change") + case <-time.After(10 * time.Millisecond): + } + + // Advance the head to the next proving period's first epoch + currentEpoch = periodStart + miner.WPoStChallengeWindow + di = miner.NewDeadlineInfo(periodStart, dlIdx, currentEpoch) + mock.setDeadline(di) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) +} + +// TestChangeHandlerStartProvingNextDeadline verifies that the proof handler +// starts proving the next deadline after the current one +func TestChangeHandlerStartProvingNextDeadline(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Trigger a head change that advances the chain beyond the submit + // confidence + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should be no change to state yet + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + di = mock.getDeadline(currentEpoch) + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Trigger head change that advances the chain to the Challenge epoch for + // the next deadline + go func() { + di = nextDeadline(di) + currentEpoch = di.Challenge + triggerHeadAdvance(t, s, currentEpoch) + }() + + // Should start generating next window's proof + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) +} + +// TestChangeHandlerProvingRounds verifies we can generate several rounds of +// proofs as the chain head advances +func TestChangeHandlerProvingRounds(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + completeProofIndex := abi.ChainEpoch(10) + for currentEpoch := abi.ChainEpoch(1); currentEpoch < miner.WPoStChallengeWindow*5; currentEpoch++ { + // Trigger a head change + di := mock.getDeadline(currentEpoch) + go triggerHeadAdvance(t, s, currentEpoch) + + // Wait for prover to process head change + <-s.ch.proveHdlr.processedHeadChanges + + completeProofEpoch := di.Open + completeProofIndex + next := nextDeadline(di) + //fmt.Println("epoch", currentEpoch, s.mock.getPostStatus(di), "next", s.mock.getPostStatus(next)) + if currentEpoch >= next.Challenge { + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + // At the next deadline's challenge epoch, should start proving + // for that epoch + require.Equal(t, postStatusProving, s.mock.getPostStatus(next)) + } else if currentEpoch > completeProofEpoch { + // After proving for the round is complete, should be in complete state + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + require.Equal(t, postStatusStart, s.mock.getPostStatus(next)) + } else { + // Until proving completes, should be in the proving state + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + require.Equal(t, postStatusStart, s.mock.getPostStatus(next)) + } + + // Wait for submitter to process head change + <-s.ch.submitHdlr.processedHeadChanges + + completeSubmitEpoch := completeProofEpoch + 1 + //fmt.Println("epoch", currentEpoch, s.submitState(di)) + if currentEpoch > completeSubmitEpoch { + require.Equal(t, SubmitStateComplete, s.submitState(di)) + } else if currentEpoch > completeProofEpoch { + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + } else { + require.Equal(t, SubmitStateStart, s.submitState(di)) + } + + if currentEpoch == completeProofEpoch { + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + } + + if currentEpoch == completeSubmitEpoch { + // Send a response to the submit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) + } + } +} + +// TestChangeHandlerProvingErrorRecovery verifies that the proof handler +// recovers correctly from an error +func TestChangeHandlerProvingErrorRecovery(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Send an error response to the call to generate proofs + mock.proveResult <- &proveRes{err: fmt.Errorf("err")} + + // Should abort and then move to start state + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusStart, s.mock.getPostStatus(di)) + + // Trigger a head change + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Send a success response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) +} + +// TestChangeHandlerSubmitErrorRecovery verifies that the submit handler +// recovers correctly from an error +func TestChangeHandlerSubmitErrorRecovery(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Read from prover incoming channel (so as not to block) + <-s.ch.proveHdlr.processedHeadChanges + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send an error response to the call to submit + mock.submitResult <- fmt.Errorf("err") + + // Should abort and then move back to the start state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateStart, s.submitState(di)) + require.True(t, mock.wasAbortCalled()) + + // Trigger another head change + go triggerHeadAdvance(t, s, currentEpoch) + + // Read from prover incoming channel (so as not to block) + <-s.ch.proveHdlr.processedHeadChanges + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send a response to the submit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerProveExpiry verifies that the prove handler +// behaves correctly on expiry +func TestChangeHandlerProveExpiry(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Move to a height that expires the current proof + currentEpoch = miner.WPoStChallengeWindow + di = mock.getDeadline(currentEpoch) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should trigger an abort and start proving for the new deadline + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + <-s.ch.proveHdlr.processedPostResults + require.True(t, mock.wasAbortCalled()) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) +} + +// TestChangeHandlerSubmitExpiry verifies that the submit handler +// behaves correctly on expiry +func TestChangeHandlerSubmitExpiry(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Move to a height that expires the submit + currentEpoch = miner.WPoStChallengeWindow + di = mock.getDeadline(currentEpoch) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should trigger an abort and move back to start state + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedSubmitResults + require.True(t, mock.wasAbortCalled()) + }() + + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + }() + + wg.Wait() +} + +// TestChangeHandlerProveRevert verifies that the prove handler +// behaves correctly on revert +func TestChangeHandlerProveRevert(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Trigger a revert to the previous epoch + revertEpoch := di.Open - 5 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + // Should be no change + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + require.False(t, mock.wasAbortCalled()) +} + +// TestChangeHandlerSubmittingRevert verifies that the submit handler +// behaves correctly when there's a revert from the submitting state +func TestChangeHandlerSubmittingRevert(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = currentEpoch + 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Trigger a revert to the previous epoch + revertEpoch := di.Open - 5 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + var wg sync.WaitGroup + wg.Add(2) + + // Should trigger an abort + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedSubmitResults + require.True(t, mock.wasAbortCalled()) + }() + + // Should resubmit current epoch + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + }() + + wg.Wait() + + // Send a response to the resubmit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerSubmitCompleteRevert verifies that the submit handler +// behaves correctly when there's a revert from the submit complete state +func TestChangeHandlerSubmitCompleteRevert(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = currentEpoch + 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send a response to the resubmit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) + + // Trigger a revert to the previous epoch + revertEpoch := di.Open - 5 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + // Should resubmit current epoch + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send a response to the resubmit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerSubmitRevertTwoEpochs verifies that the submit handler +// behaves correctly when the revert is two epochs deep +func TestChangeHandlerSubmitRevertTwoEpochs(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + diE1 := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(diE1)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: diE1.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE1)) + + // Move to the challenge epoch for the next deadline + diE2 := nextDeadline(diE1) + currentEpoch = diE2.Challenge + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state for epoch 1 + <-s.ch.submitHdlr.processedHeadChanges + diE1 = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(diE1)) + + // Send a response to the submit call for epoch 1 + mock.submitResult <- nil + + // Should move to the complete state for epoch 1 + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(diE1)) + + // Should start proving epoch 2 + // Send a response to the call to generate proofs + postsE2 := []miner.SubmitWindowedPoStParams{{Deadline: diE2.Index}} + mock.proveResult <- &proveRes{posts: postsE2} + + // Should move to proving complete for epoch 2 + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE2)) + + // Move to the correct height to submit the proof for epoch 2 + currentEpoch = diE2.Open + 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state for epoch 2 + <-s.ch.submitHdlr.processedHeadChanges + diE2 = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(diE2)) + + // Trigger a revert through two epochs (from epoch 2 to epoch 0) + revertEpoch := diE1.Open - 5 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + var wg sync.WaitGroup + wg.Add(2) + + // Should trigger an abort + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedSubmitResults + require.True(t, mock.wasAbortCalled()) + }() + + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedHeadChanges + + // Should reset epoch 1 (that is expired) to start state + require.Equal(t, SubmitStateStart, s.submitState(diE1)) + // Should resubmit epoch 2 + require.Equal(t, SubmitStateSubmitting, s.submitState(diE2)) + }() + + wg.Wait() + + // Send a response to the resubmit call for epoch 2 + mock.submitResult <- nil + + // Should move to the complete state for epoch 2 + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(diE2)) +} + +// TestChangeHandlerSubmitRevertAdvanceLess verifies that the submit handler +// behaves correctly when the revert is two epochs deep and the advance is +// to a lower height than before +func TestChangeHandlerSubmitRevertAdvanceLess(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + diE1 := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(diE1)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: diE1.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE1)) + + // Move to the challenge epoch for the next deadline + diE2 := nextDeadline(diE1) + currentEpoch = diE2.Challenge + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state for epoch 1 + <-s.ch.submitHdlr.processedHeadChanges + diE1 = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(diE1)) + + // Send a response to the submit call for epoch 1 + mock.submitResult <- nil + + // Should move to the complete state for epoch 1 + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(diE1)) + + // Should start proving epoch 2 + // Send a response to the call to generate proofs + postsE2 := []miner.SubmitWindowedPoStParams{{Deadline: diE2.Index}} + mock.proveResult <- &proveRes{posts: postsE2} + + // Should move to proving complete for epoch 2 + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE2)) + + // Move to the correct height to submit the proof for epoch 2 + currentEpoch = diE2.Open + 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state for epoch 2 + <-s.ch.submitHdlr.processedHeadChanges + diE2 = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(diE2)) + + // Trigger a revert through two epochs (from epoch 2 to epoch 0) + // then advance to the previous epoch (to epoch 1) + revertEpoch := diE1.Open - 5 + currentEpoch = diE2.Open - 1 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + var wg sync.WaitGroup + wg.Add(2) + + // Should trigger an abort + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedSubmitResults + require.True(t, mock.wasAbortCalled()) + }() + + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedHeadChanges + + // Should resubmit epoch 1 + require.Equal(t, SubmitStateSubmitting, s.submitState(diE1)) + // Should reset epoch 2 to start state + require.Equal(t, SubmitStateStart, s.submitState(diE2)) + }() + + wg.Wait() + + // Send a response to the resubmit call for epoch 1 + mock.submitResult <- nil + + // Should move to the complete state for epoch 1 + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(diE1)) +} + +type smScaffolding struct { + ctx context.Context + mock *mockAPI + ch *changeHandler +} + +func makeScaffolding(t *testing.T) *smScaffolding { + ctx := context.Background() + actor := tutils.NewActorAddr(t, "actor") + mock := newMockAPI() + ch := newChangeHandler(mock, actor) + mock.setChangeHandler(ch) + + ch.proveHdlr.processedHeadChanges = make(chan *headChange) + ch.proveHdlr.processedPostResults = make(chan *postResult) + + ch.submitHdlr.processedHeadChanges = make(chan *headChange) + ch.submitHdlr.processedSubmitResults = make(chan *submitResult) + + return &smScaffolding{ + ctx: ctx, + mock: mock, + ch: ch, + } +} + +func triggerHeadAdvance(t *testing.T, s *smScaffolding, height abi.ChainEpoch) { + ts := s.mock.makeTs(t, height) + err := s.ch.update(s.ctx, nil, ts) + require.NoError(t, err) +} + +func triggerHeadChange(t *testing.T, s *smScaffolding, revertHeight, advanceHeight abi.ChainEpoch) { + tsRev := s.mock.makeTs(t, revertHeight) + tsAdv := s.mock.makeTs(t, advanceHeight) + err := s.ch.update(s.ctx, tsRev, tsAdv) + require.NoError(t, err) +} + +func (s *smScaffolding) submitState(di *dline.Info) SubmitState { + return s.ch.submitHdlr.getPostWindow(di).submitState +} + +func makeTs(t *testing.T, h abi.ChainEpoch) *types.TipSet { + var parents []cid.Cid + msgcid := dummyCid + + a, _ := address.NewFromString("t00") + b, _ := address.NewFromString("t02") + var ts, err = types.NewTipSet([]*types.BlockHeader{ + { + Height: h, + Miner: a, + + Parents: parents, + + Ticket: &types.Ticket{VRFProof: []byte{byte(h % 2)}}, + + ParentStateRoot: dummyCid, + Messages: msgcid, + ParentMessageReceipts: dummyCid, + + BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, + BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, + }, + { + Height: h, + Miner: b, + + Parents: parents, + + Ticket: &types.Ticket{VRFProof: []byte{byte((h + 1) % 2)}}, + + ParentStateRoot: dummyCid, + Messages: msgcid, + ParentMessageReceipts: dummyCid, + + BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, + BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, + }, + }) + + require.NoError(t, err) + + return ts +} diff --git a/storage/wdpost_nextdl_test.go b/storage/wdpost_nextdl_test.go new file mode 100644 index 000000000..ad4b1fdeb --- /dev/null +++ b/storage/wdpost_nextdl_test.go @@ -0,0 +1,38 @@ +package storage + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +func TestNextDeadline(t *testing.T) { + periodStart := abi.ChainEpoch(0) + deadlineIdx := 0 + currentEpoch := abi.ChainEpoch(10) + + di := miner.NewDeadlineInfo(periodStart, uint64(deadlineIdx), currentEpoch) + require.EqualValues(t, 0, di.Index) + require.EqualValues(t, 0, di.PeriodStart) + require.EqualValues(t, -20, di.Challenge) + require.EqualValues(t, 0, di.Open) + require.EqualValues(t, 60, di.Close) + + for i := 1; i < 1+int(miner.WPoStPeriodDeadlines)*2; i++ { + di = nextDeadline(di) + deadlineIdx = i % int(miner.WPoStPeriodDeadlines) + expPeriodStart := int(miner.WPoStProvingPeriod) * (i / int(miner.WPoStPeriodDeadlines)) + expOpen := expPeriodStart + deadlineIdx*int(miner.WPoStChallengeWindow) + expClose := expOpen + int(miner.WPoStChallengeWindow) + expChallenge := expOpen - int(miner.WPoStChallengeLookback) + //fmt.Printf("%d: %d@%d %d-%d (%d)\n", i, expPeriodStart, deadlineIdx, expOpen, expClose, expChallenge) + require.EqualValues(t, deadlineIdx, di.Index) + require.EqualValues(t, expPeriodStart, di.PeriodStart) + require.EqualValues(t, expOpen, di.Open) + require.EqualValues(t, expClose, di.Close) + require.EqualValues(t, expChallenge, di.Challenge) + } +} diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 9a497f879..35fdfc6d1 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -29,15 +29,21 @@ import ( "github.com/filecoin-project/lotus/journal" ) -func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) { +func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dline.Info) { journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + c := evtCommon{Error: err} + if ts != nil { + c.Deadline = deadline + c.Height = ts.Height() + c.TipSet = ts.Cids() + } return WdPoStSchedulerEvt{ - evtCommon: s.getEvtCommon(err), + evtCommon: c, State: SchedulerStateFaulted, } }) - log.Errorf("TODO") + log.Errorf("Got err %w - TODO handle errors", err) /*s.failLk.Lock() if eps > s.failed { s.failed = eps @@ -45,67 +51,134 @@ func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) { s.failLk.Unlock()*/ } -func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, ts *types.TipSet) { - ctx, abort := context.WithCancel(ctx) - - s.abort = abort - s.activeDeadline = deadline - - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { - return WdPoStSchedulerEvt{ - evtCommon: s.getEvtCommon(nil), - State: SchedulerStateStarted, +// recordProofsEvent records a successful proofs_processed event in the +// journal, even if it was a noop (no partitions). +func (s *WindowPoStScheduler) recordProofsEvent(partitions []miner.PoStPartition, mcid cid.Cid) { + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { + return &WdPoStProofsProcessedEvt{ + evtCommon: s.getEvtCommon(nil), + Partitions: partitions, + MessageCID: mcid, } }) +} +// startGeneratePoST kicks off the process of generating a PoST +func (s *WindowPoStScheduler) startGeneratePoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + completeGeneratePoST CompleteGeneratePoSTCb, +) context.CancelFunc { + ctx, abort := context.WithCancel(ctx) go func() { defer abort() - ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.doPost") - defer span.End() - - // recordProofsEvent records a successful proofs_processed event in the - // journal, even if it was a noop (no partitions). - recordProofsEvent := func(partitions []miner.PoStPartition, mcid cid.Cid) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { - return &WdPoStProofsProcessedEvt{ - evtCommon: s.getEvtCommon(nil), - Partitions: partitions, - MessageCID: mcid, - } - }) - } - - posts, err := s.runPost(ctx, *deadline, ts) - if err != nil { - log.Errorf("run window post failed: %+v", err) - s.failPost(err, deadline) - return - } - - if len(posts) == 0 { - recordProofsEvent(nil, cid.Undef) - return - } - - for i := range posts { - post := &posts[i] - sm, err := s.submitPost(ctx, post) - if err != nil { - log.Errorf("submit window post failed: %+v", err) - s.failPost(err, deadline) - } else { - recordProofsEvent(post.Partitions, sm.Cid()) - } - } - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ evtCommon: s.getEvtCommon(nil), - State: SchedulerStateSucceeded, + State: SchedulerStateStarted, } }) + + posts, err := s.runGeneratePoST(ctx, ts, deadline) + completeGeneratePoST(posts, err) }() + + return abort +} + +// runGeneratePoST generates the PoST +func (s *WindowPoStScheduler) runGeneratePoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, +) ([]miner.SubmitWindowedPoStParams, error) { + ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.generatePoST") + defer span.End() + + posts, err := s.runPost(ctx, *deadline, ts) + if err != nil { + log.Errorf("runPost failed: %+v", err) + return nil, err + } + + if len(posts) == 0 { + s.recordProofsEvent(nil, cid.Undef) + } + + return posts, nil +} + +// startSubmitPoST kicks of the process of submitting PoST +func (s *WindowPoStScheduler) startSubmitPoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + posts []miner.SubmitWindowedPoStParams, + completeSubmitPoST CompleteSubmitPoSTCb, +) context.CancelFunc { + + ctx, abort := context.WithCancel(ctx) + go func() { + defer abort() + + err := s.runSubmitPoST(ctx, ts, deadline, posts) + if err == nil { + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + return WdPoStSchedulerEvt{ + evtCommon: s.getEvtCommon(nil), + State: SchedulerStateSucceeded, + } + }) + } + completeSubmitPoST(err) + }() + + return abort +} + +// runSubmitPoST submits PoST +func (s *WindowPoStScheduler) runSubmitPoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + posts []miner.SubmitWindowedPoStParams, +) error { + if len(posts) == 0 { + return nil + } + + ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.submitPoST") + defer span.End() + + // Get randomness from tickets + commEpoch := deadline.Open + commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) + if err != nil { + err = xerrors.Errorf("failed to get chain randomness from tickets for windowPost (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) + log.Errorf("submitPost failed: %+v", err) + + return err + } + + var submitErr error + for i := range posts { + // Add randomness to PoST + post := &posts[i] + post.ChainCommitEpoch = commEpoch + post.ChainCommitRand = commRand + + // Submit PoST + sm, submitErr := s.submitPost(ctx, post) + if submitErr != nil { + log.Errorf("submit window post failed: %+v", submitErr) + } else { + s.recordProofsEvent(post.Partitions, sm.Cid()) + } + } + + return submitErr } func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.BitField) (bitfield.BitField, error) { @@ -392,7 +465,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty rand, err := s.api.ChainGetRandomnessFromBeacon(ctx, ts.Key(), crypto.DomainSeparationTag_WindowedPoStChallengeSeed, di.Challenge, buf.Bytes()) if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err) + return nil, xerrors.Errorf("failed to get chain randomness from beacon for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err) } // Get the partitions for the given deadline @@ -536,19 +609,6 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty posts = append(posts, params) } - // Compute randomness after generating proofs so as to reduce the impact - // of chain reorgs (which change randomness) - commEpoch := di.Open - commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) - if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) - } - - for i := range posts { - posts[i].ChainCommitEpoch = commEpoch - posts[i].ChainCommitRand = commRand - } - return posts, nil } @@ -589,6 +649,7 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]a } batches = append(batches, partitions[i:end]) } + return batches, nil } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 10be2fbcd..09b9aee5c 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" @@ -177,7 +178,10 @@ func TestWDPostDoPost(t *testing.T) { FaultDeclarationCutoff: miner0.FaultDeclarationCutoff, } ts := mockTipSet(t) - scheduler.doPost(ctx, di, ts) + + scheduler.startGeneratePoST(ctx, ts, di, func(posts []miner.SubmitWindowedPoStParams, err error) { + scheduler.startSubmitPoST(ctx, ts, di, posts, func(err error) {}) + }) // Read the window PoST messages for i := 0; i < expectedMsgCount; i++ { diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 7e60fd9ee..3a76a219f 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -22,8 +22,6 @@ import ( "go.opencensus.io/trace" ) -const StartConfidence = 4 // TODO: config - type WindowPoStScheduler struct { api storageMinerApi feeCfg config.MinerFeeConfig @@ -31,16 +29,11 @@ type WindowPoStScheduler struct { faultTracker sectorstorage.FaultTracker proofType abi.RegisteredPoStProof partitionSectors uint64 + ch *changeHandler actor address.Address worker address.Address - cur *types.TipSet - - // if a post is in progress, this indicates for which ElectionPeriodStart - activeDeadline *dline.Info - abort context.CancelFunc - evtTypes [4]journal.EventType // failed abi.ChainEpoch // eps @@ -77,16 +70,17 @@ func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb }, nil } -func deadlineEquals(a, b *dline.Info) bool { - if a == nil || b == nil { - return b == a - } - - return a.PeriodStart == b.PeriodStart && a.Index == b.Index && a.Challenge == b.Challenge +type changeHandlerAPIImpl struct { + storageMinerApi + *WindowPoStScheduler } func (s *WindowPoStScheduler) Run(ctx context.Context) { - defer s.abortActivePoSt() + // Initialize change handler + chImpl := &changeHandlerAPIImpl{storageMinerApi: s.api, WindowPoStScheduler: s} + s.ch = newChangeHandler(chImpl, s.actor) + defer s.ch.shutdown() + s.ch.start() var notifs <-chan []*api.HeadChange var err error @@ -125,9 +119,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { continue } - if err := s.update(ctx, chg.Val); err != nil { - log.Errorf("%+v", err) - } + s.update(ctx, nil, chg.Val) gotCur = true continue @@ -135,7 +127,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.headChange") - var lowest, highest *types.TipSet = s.cur, nil + var lowest, highest *types.TipSet = nil, nil for _, change := range changes { if change.Val == nil { @@ -149,12 +141,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { } } - if err := s.revert(ctx, lowest); err != nil { - log.Error("handling head reverts in window post sched: %+v", err) - } - if err := s.update(ctx, highest); err != nil { - log.Error("handling head updates in window post sched: %+v", err) - } + s.update(ctx, lowest, highest) span.End() case <-ctx.Done(): @@ -163,95 +150,40 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { } } -func (s *WindowPoStScheduler) revert(ctx context.Context, newLowest *types.TipSet) error { - if s.cur == newLowest { - return nil +func (s *WindowPoStScheduler) update(ctx context.Context, revert, apply *types.TipSet) { + if apply == nil { + log.Error("no new tipset in window post WindowPoStScheduler.update") + return } - s.cur = newLowest - - newDeadline, err := s.api.StateMinerProvingDeadline(ctx, s.actor, newLowest.Key()) + err := s.ch.update(ctx, revert, apply) if err != nil { - return err + log.Errorf("handling head updates in window post sched: %+v", err) } - - if !deadlineEquals(s.activeDeadline, newDeadline) { - s.abortActivePoSt() - } - - return nil } -func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) error { - if new == nil { - return xerrors.Errorf("no new tipset in window post sched update") - } - - di, err := s.api.StateMinerProvingDeadline(ctx, s.actor, new.Key()) - if err != nil { - return err - } - - if deadlineEquals(s.activeDeadline, di) { - return nil // already working on this deadline - } - - if !di.PeriodStarted() { - return nil // not proving anything yet - } - - s.abortActivePoSt() - - // TODO: wait for di.Challenge here, will give us ~10min more to compute windowpost - // (Need to get correct deadline above, which is tricky) - - if di.Open+StartConfidence >= new.Height() { - log.Info("not starting window post yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height()) - return nil - } - - /*s.failLk.Lock() - if s.failed > 0 { - s.failed = 0 - s.activeEPS = 0 - } - s.failLk.Unlock()*/ - log.Infof("at %d, do window post for P %d, dd %d", new.Height(), di.PeriodStart, di.Index) - - s.doPost(ctx, di, new) - - return nil +// onAbort is called when generating proofs or submitting proofs is aborted +func (s *WindowPoStScheduler) onAbort(ts *types.TipSet, deadline *dline.Info) { + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + c := evtCommon{} + if ts != nil { + c.Deadline = deadline + c.Height = ts.Height() + c.TipSet = ts.Cids() + } + return WdPoStSchedulerEvt{ + evtCommon: c, + State: SchedulerStateAborted, + } + }) } -func (s *WindowPoStScheduler) abortActivePoSt() { - if s.activeDeadline == nil { - return // noop - } - - if s.abort != nil { - s.abort() - - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { - return WdPoStSchedulerEvt{ - evtCommon: s.getEvtCommon(nil), - State: SchedulerStateAborted, - } - }) - - log.Warnf("Aborting window post (Deadline: %+v)", s.activeDeadline) - } - - s.activeDeadline = nil - s.abort = nil -} - -// getEvtCommon populates and returns common attributes from state, for a -// WdPoSt journal event. func (s *WindowPoStScheduler) getEvtCommon(err error) evtCommon { c := evtCommon{Error: err} - if s.cur != nil { - c.Deadline = s.activeDeadline - c.Height = s.cur.Height() - c.TipSet = s.cur.Cids() + currentTS, currentDeadline := s.ch.currentTSDI() + if currentTS != nil { + c.Deadline = currentDeadline + c.Height = currentTS.Height() + c.TipSet = currentTS.Cids() } return c } From e48c525053ada752041c4917a5c22d9610e8746d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 24 Sep 2020 19:41:14 +0200 Subject: [PATCH 17/80] Re-add jaeger-tracing --- documentation/en/jaeger-tracing.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 documentation/en/jaeger-tracing.md diff --git a/documentation/en/jaeger-tracing.md b/documentation/en/jaeger-tracing.md new file mode 100644 index 000000000..bbe4d3052 --- /dev/null +++ b/documentation/en/jaeger-tracing.md @@ -0,0 +1,26 @@ +# Jaeger Tracing + +Lotus has tracing built into many of its internals. To view the traces, first download [Jaeger](https://www.jaegertracing.io/download/) (Choose the 'all-in-one' binary). Then run it somewhere, start up the lotus daemon, and open up localhost:16686 in your browser. + +## Open Census + +Lotus uses [OpenCensus](https://opencensus.io/) for tracing application flow. This generates spans through the execution of annotated code paths. + +Currently it is set up to use Jaeger, though other tracing backends should be fairly easy to swap in. + +## Running Locally + +To easily run and view tracing locally, first, install jaeger. The easiest way to do this is to [download the binaries](https://www.jaegertracing.io/download/) and then run the `jaeger-all-in-one` binary. This will start up jaeger, listen for spans on `localhost:6831`, and expose a web UI for viewing traces on `http://localhost:16686/`. + +Now, to start sending traces from Lotus to Jaeger, set the environment variable `LOTUS_JAEGER` to `localhost:6831`, and start the `lotus daemon`. + +Now, to view any generated traces, open up `http://localhost:16686/` in your browser. + +## Adding Spans + +To annotate a new codepath with spans, add the following lines to the top of the function you wish to trace: + +```go +ctx, span := trace.StartSpan(ctx, "put function name here") +defer span.End() +``` From 13b38815c2e6df6d7d16cb024f3fedad1b2a4967 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Fri, 25 Sep 2020 20:31:07 +0800 Subject: [PATCH 18/80] add trace wdpost --- storage/wdpost_sched.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 7e60fd9ee..9477e0306 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -125,10 +125,13 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { continue } + ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.headChange") + if err := s.update(ctx, chg.Val); err != nil { log.Errorf("%+v", err) } + span.End() gotCur = true continue } From 80a7ed811638853987a3124175606d7db5269bf2 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 25 Sep 2020 15:54:27 +0200 Subject: [PATCH 19/80] refactor: use abstract types instead of specs-actors --- chain/actors/builtin/miner/miner.go | 4 ++++ storage/wdpost_changehandler.go | 8 ++++++-- storage/wdpost_changehandler_test.go | 8 ++++---- storage/wdpost_nextdl_test.go | 4 ++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 50a0fc5ca..e09cac587 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -21,6 +21,10 @@ import ( // Unchanged between v0 and v1 actors var PreCommitChallengeDelay = miner0.PreCommitChallengeDelay var WPoStProvingPeriod = miner0.WPoStProvingPeriod +var WPoStPeriodDeadlines = miner0.WPoStPeriodDeadlines +var WPoStChallengeWindow = miner0.WPoStChallengeWindow +var WPoStChallengeLookback = miner0.WPoStChallengeLookback +var FaultDeclarationCutoff = miner0.FaultDeclarationCutoff const MinSectorExpiration = miner0.MinSectorExpiration diff --git a/storage/wdpost_changehandler.go b/storage/wdpost_changehandler.go index e65b7a7fc..285995757 100644 --- a/storage/wdpost_changehandler.go +++ b/storage/wdpost_changehandler.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/chain/types" @@ -529,5 +529,9 @@ func nextDeadline(currentDeadline *dline.Info) *dline.Info { periodStart = periodStart + miner.WPoStProvingPeriod } - return miner.NewDeadlineInfo(periodStart, newDeadline, currentDeadline.CurrentEpoch) + return NewDeadlineInfo(periodStart, newDeadline, currentDeadline.CurrentEpoch) +} + +func NewDeadlineInfo(periodStart abi.ChainEpoch, deadlineIdx uint64, currEpoch abi.ChainEpoch) *dline.Info { + return dline.NewInfo(periodStart, deadlineIdx, currEpoch, miner.WPoStPeriodDeadlines, miner.WPoStProvingPeriod, miner.WPoStChallengeWindow, miner.WPoStChallengeLookback, miner.FaultDeclarationCutoff) } diff --git a/storage/wdpost_changehandler_test.go b/storage/wdpost_changehandler_test.go index d2a4779e6..6479c0d7e 100644 --- a/storage/wdpost_changehandler_test.go +++ b/storage/wdpost_changehandler_test.go @@ -17,8 +17,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) var dummyCid cid.Cid @@ -90,7 +90,7 @@ func (m *mockAPI) getDeadline(currentEpoch abi.ChainEpoch) *dline.Info { close += miner.WPoStChallengeWindow dlIdx++ } - return miner.NewDeadlineInfo(0, dlIdx, currentEpoch) + return NewDeadlineInfo(0, dlIdx, currentEpoch) } func (m *mockAPI) StateMinerProvingDeadline(ctx context.Context, address address.Address, key types.TipSetKey) (*dline.Info, error) { @@ -355,7 +355,7 @@ func TestChangeHandlerDontStartUntilProvingPeriod(t *testing.T) { periodStart := miner.WPoStProvingPeriod dlIdx := uint64(1) currentEpoch := abi.ChainEpoch(10) - di := miner.NewDeadlineInfo(periodStart, dlIdx, currentEpoch) + di := NewDeadlineInfo(periodStart, dlIdx, currentEpoch) mock.setDeadline(di) defer s.ch.shutdown() @@ -375,7 +375,7 @@ func TestChangeHandlerDontStartUntilProvingPeriod(t *testing.T) { // Advance the head to the next proving period's first epoch currentEpoch = periodStart + miner.WPoStChallengeWindow - di = miner.NewDeadlineInfo(periodStart, dlIdx, currentEpoch) + di = NewDeadlineInfo(periodStart, dlIdx, currentEpoch) mock.setDeadline(di) go triggerHeadAdvance(t, s, currentEpoch) diff --git a/storage/wdpost_nextdl_test.go b/storage/wdpost_nextdl_test.go index ad4b1fdeb..4a23bad65 100644 --- a/storage/wdpost_nextdl_test.go +++ b/storage/wdpost_nextdl_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" ) func TestNextDeadline(t *testing.T) { @@ -14,7 +14,7 @@ func TestNextDeadline(t *testing.T) { deadlineIdx := 0 currentEpoch := abi.ChainEpoch(10) - di := miner.NewDeadlineInfo(periodStart, uint64(deadlineIdx), currentEpoch) + di := NewDeadlineInfo(periodStart, uint64(deadlineIdx), currentEpoch) require.EqualValues(t, 0, di.Index) require.EqualValues(t, 0, di.PeriodStart) require.EqualValues(t, -20, di.Challenge) From 650725d71e909089e5087169d8d694e3f4076e4c Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 26 Sep 2020 17:30:58 +0800 Subject: [PATCH 20/80] add printf --- api/test/window_post.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 683489a91..958c91816 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -153,18 +153,16 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) fmt.Printf("Running one proving period\n") + fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2) for { head, err := client.ChainHead(ctx) require.NoError(t, err) if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 { + fmt.Printf("Now head.Height = %d\n", head.Height()) break } - - if head.Height()%100 == 0 { - fmt.Printf("@%d\n", head.Height()) - } build.Clock.Sleep(blocktime) } @@ -186,7 +184,6 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.Greater(t, len(parts), 0) secs := parts[0].AllSectors - require.NoError(t, err) n, err := secs.Count() require.NoError(t, err) require.Equal(t, uint64(2), n) @@ -210,7 +207,6 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.Greater(t, len(parts), 0) secs := parts[0].AllSectors - require.NoError(t, err) n, err := secs.Count() require.NoError(t, err) require.Equal(t, uint64(2), n) @@ -236,18 +232,17 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) fmt.Printf("Go through another PP, wait for sectors to become faulty\n") + fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2) for { head, err := client.ChainHead(ctx) require.NoError(t, err) if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 { + fmt.Printf("Now head.Height = %d\n", head.Height()) break } - if head.Height()%100 == 0 { - fmt.Printf("@%d\n", head.Height()) - } build.Clock.Sleep(blocktime) } @@ -267,17 +262,17 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) require.NoError(t, err) + fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2) + for { head, err := client.ChainHead(ctx) require.NoError(t, err) if head.Height() > di.PeriodStart+di.WPoStProvingPeriod+2 { + fmt.Printf("Now head.Height = %d\n", head.Height()) break } - if head.Height()%100 == 0 { - fmt.Printf("@%d\n", head.Height()) - } build.Clock.Sleep(blocktime) } @@ -300,12 +295,14 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) waitUntil := head.Height() + 10 + fmt.Printf("End for head.Height > %d\n", waitUntil) for { head, err := client.ChainHead(ctx) require.NoError(t, err) if head.Height() > waitUntil { + fmt.Printf("Now head.Height = %d\n", head.Height()) break } } From 8955b8d8a7ae3110acd82f072e9d739be7f73ac1 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 26 Sep 2020 21:16:28 +0200 Subject: [PATCH 21/80] Centralize filtering, output wallet addresses --- cmd/lotus-shed/dealtracker.go | 186 ++++++++++++++++++---------------- 1 file changed, 97 insertions(+), 89 deletions(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index d39f51bd1..a21923009 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -5,8 +5,7 @@ import ( "encoding/json" "net" "net/http" - "os" - "strings" + "sync" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" @@ -19,23 +18,27 @@ type dealStatsServer struct { api api.FullNode } -var filteredClients map[address.Address]bool +// these lists grow continuously with the network +// TODO: need to switch this to an LRU of sorts, to ensure refreshes +var knownFiltered = new(sync.Map) +var resolvedWallets = new(sync.Map) func init() { - fc := []string{"t0112", "t0113", "t0114", "t010089"} - - filtered, set := os.LookupEnv("FILTERED_CLIENTS") - if set { - fc = strings.Split(filtered, ":") - } - - filteredClients = make(map[address.Address]bool) - for _, a := range fc { - addr, err := address.NewFromString(a) + for _, a := range []string{ + "t0100", // client for genesis miner + "t0112", // client for genesis miner + "t0113", // client for genesis miner + "t0114", // client for genesis miner + "t1nslxql4pck5pq7hddlzym3orxlx35wkepzjkm3i", // SR1 dealbot wallet + "t1stghxhdp2w53dym2nz2jtbpk6ccd4l2lxgmezlq", // SR1 dealbot wallet + "t1mcr5xkgv4jdl3rnz77outn6xbmygb55vdejgbfi", // SR1 dealbot wallet + "t1qiqdbbmrdalbntnuapriirduvxu5ltsc5mhy7si", // SR1 dealbot wallet + } { + a, err := address.NewFromString(a) if err != nil { panic(err) } - filteredClients[addr] = true + knownFiltered.Store(a, true) } } @@ -45,32 +48,16 @@ type dealCountResp struct { } func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) { - ctx := context.Background() - head, err := dss.api.ChainHead(ctx) - if err != nil { - log.Warnf("failed to get chain head: %s", err) + epoch, deals := dss.filteredDealList() + if epoch == 0 { w.WriteHeader(500) return } - deals, err := dss.api.StateMarketDeals(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get market deals: %s", err) - w.WriteHeader(500) - return - } - - var count int64 - for _, d := range deals { - if !filteredClients[d.Proposal.Client] { - count++ - } - } - if err := json.NewEncoder(w).Encode(&dealCountResp{ - Total: count, - Epoch: int64(head.Height()), + Total: int64(len(deals)), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal count response: %s", err) return @@ -83,34 +70,21 @@ type dealAverageResp struct { } func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) { - ctx := context.Background() - head, err := dss.api.ChainHead(ctx) - if err != nil { - log.Warnf("failed to get chain head: %s", err) + epoch, deals := dss.filteredDealList() + if epoch == 0 { w.WriteHeader(500) return } - deals, err := dss.api.StateMarketDeals(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get market deals: %s", err) - w.WriteHeader(500) - return - } - - var count int64 var totalBytes int64 for _, d := range deals { - if !filteredClients[d.Proposal.Client] { - count++ - totalBytes += int64(d.Proposal.PieceSize.Unpadded()) - } + totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded()) } if err := json.NewEncoder(w).Encode(&dealAverageResp{ - AverageSize: totalBytes / count, - Epoch: int64(head.Height()), + AverageSize: totalBytes / int64(len(deals)), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal average response: %s", err) return @@ -123,32 +97,20 @@ type dealTotalResp struct { } func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) { - ctx := context.Background() - - head, err := dss.api.ChainHead(ctx) - if err != nil { - log.Warnf("failed to get chain head: %s", err) - w.WriteHeader(500) - return - } - - deals, err := dss.api.StateMarketDeals(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get market deals: %s", err) + epoch, deals := dss.filteredDealList() + if epoch == 0 { w.WriteHeader(500) return } var totalBytes int64 for _, d := range deals { - if !filteredClients[d.Proposal.Client] { - totalBytes += int64(d.Proposal.PieceSize.Unpadded()) - } + totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded()) } if err := json.NewEncoder(w).Encode(&dealTotalResp{ TotalBytes: totalBytes, - Epoch: int64(head.Height()), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal average response: %s", err) return @@ -168,18 +130,8 @@ type clientStatsOutput struct { } func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *http.Request) { - ctx := context.Background() - - head, err := dss.api.ChainHead(ctx) - if err != nil { - log.Warnf("failed to get chain head: %s", err) - w.WriteHeader(500) - return - } - - deals, err := dss.api.StateMarketDeals(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get market deals: %s", err) + epoch, deals := dss.filteredDealList() + if epoch == 0 { w.WriteHeader(500) return } @@ -187,23 +139,20 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h stats := make(map[address.Address]*clientStatsOutput) for _, d := range deals { - if filteredClients[d.Proposal.Client] { - continue - } - st, ok := stats[d.Proposal.Client] + st, ok := stats[d.deal.Proposal.Client] if !ok { st = &clientStatsOutput{ - Client: d.Proposal.Client, + Client: d.resolvedWallet, cids: make(map[cid.Cid]bool), providers: make(map[address.Address]bool), } - stats[d.Proposal.Client] = st + stats[d.deal.Proposal.Client] = st } - st.DataSize += int64(d.Proposal.PieceSize.Unpadded()) - st.cids[d.Proposal.PieceCID] = true - st.providers[d.Proposal.Provider] = true + st.DataSize += int64(d.deal.Proposal.PieceSize.Unpadded()) + st.cids[d.deal.Proposal.PieceCID] = true + st.providers[d.deal.Proposal.Provider] = true st.NumDeals++ } @@ -221,6 +170,65 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h } } +type dealInfo struct { + deal api.MarketDeal + resolvedWallet address.Address +} + +// filteredDealList returns the current epoch and a list of filtered deals +// on error returns an epoch of 0 +func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) { + ctx := context.Background() + + head, err := dss.api.ChainHead(ctx) + if err != nil { + log.Warnf("failed to get chain head: %s", err) + return 0, nil + } + + deals, err := dss.api.StateMarketDeals(ctx, head.Key()) + if err != nil { + log.Warnf("failed to get market deals: %s", err) + return 0, nil + } + + ret := make(map[string]dealInfo, len(deals)) + for dealKey, d := range deals { + + // Counting no-longer-active deals as per Pooja's request + // // https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85 + // if d.State.SectorStartEpoch < 0 { + // continue + // } + + if _, isFiltered := knownFiltered.Load(d.Proposal.Client); isFiltered { + continue + } + + if _, wasSeen := resolvedWallets.Load(d.Proposal.Client); !wasSeen { + w, err := dss.api.StateAccountKey(ctx, d.Proposal.Client, head.Key()) + if err != nil { + log.Warnf("failed to resolve id '%s' to wallet address: %s", d.Proposal.Client, err) + continue + } else { + resolvedWallets.Store(d.Proposal.Client, w) + } + } + + w, _ := resolvedWallets.Load(d.Proposal.Client) + if _, isFiltered := knownFiltered.Load(w); isFiltered { + continue + } + + ret[dealKey] = dealInfo{ + deal: d, + resolvedWallet: w.(address.Address), + } + } + + return int64(head.Height()), ret +} + var serveDealStatsCmd = &cli.Command{ Name: "serve-deal-stats", Flags: []cli.Flag{}, From 10cdbadd82158cc36959877733ad008188b6aa14 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 26 Sep 2020 21:29:11 +0200 Subject: [PATCH 22/80] Arrange json as the frontend expects it --- cmd/lotus-shed/dealtracker.go | 55 ++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index a21923009..1340dc9c6 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -43,8 +43,9 @@ func init() { } type dealCountResp struct { - Total int64 `json:"total"` - Epoch int64 `json:"epoch"` + Epoch int64 `json:"epoch"` + Endpoint string `json:"endpoint"` + Payload int64 `json:"payload"` } func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) { @@ -56,8 +57,9 @@ func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *htt } if err := json.NewEncoder(w).Encode(&dealCountResp{ - Total: int64(len(deals)), - Epoch: epoch, + Endpoint: "COUNT_DEALS", + Payload: int64(len(deals)), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal count response: %s", err) return @@ -65,8 +67,9 @@ func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *htt } type dealAverageResp struct { - AverageSize int64 `json:"average_size"` - Epoch int64 `json:"epoch"` + Epoch int64 `json:"epoch"` + Endpoint string `json:"endpoint"` + Payload int64 `json:"payload"` } func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) { @@ -83,8 +86,9 @@ func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, } if err := json.NewEncoder(w).Encode(&dealAverageResp{ - AverageSize: totalBytes / int64(len(deals)), - Epoch: epoch, + Endpoint: "AVERAGE_DEAL_SIZE", + Payload: totalBytes / int64(len(deals)), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal average response: %s", err) return @@ -92,8 +96,9 @@ func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, } type dealTotalResp struct { - TotalBytes int64 `json:"total_size"` - Epoch int64 `json:"epoch"` + Epoch int64 `json:"epoch"` + Endpoint string `json:"endpoint"` + Payload int64 `json:"payload"` } func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) { @@ -109,8 +114,9 @@ func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r } if err := json.NewEncoder(w).Encode(&dealTotalResp{ - TotalBytes: totalBytes, - Epoch: epoch, + Endpoint: "DEAL_BYTES", + Payload: totalBytes, + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal average response: %s", err) return @@ -119,6 +125,12 @@ func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r } type clientStatsOutput struct { + Epoch int64 `json:"epoch"` + Endpoint string `json:"endpoint"` + Payload []*clientStats `json:"payload"` +} + +type clientStats struct { Client address.Address `json:"client"` DataSize int64 `json:"data_size"` NumCids int `json:"num_cids"` @@ -136,13 +148,13 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h return } - stats := make(map[address.Address]*clientStatsOutput) + stats := make(map[address.Address]*clientStats) for _, d := range deals { st, ok := stats[d.deal.Proposal.Client] if !ok { - st = &clientStatsOutput{ + st = &clientStats{ Client: d.resolvedWallet, cids: make(map[cid.Cid]bool), providers: make(map[address.Address]bool), @@ -156,12 +168,15 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h st.NumDeals++ } - out := make([]*clientStatsOutput, 0, len(stats)) - for _, cso := range stats { - cso.NumCids = len(cso.cids) - cso.NumMiners = len(cso.providers) - - out = append(out, cso) + out := clientStatsOutput{ + Epoch: epoch, + Endpoint: "CLIENT_DEAL_STATS", + Payload: make([]*clientStats, 0, len(stats)), + } + for _, cs := range stats { + cs.NumCids = len(cs.cids) + cs.NumMiners = len(cs.providers) + out.Payload = append(out.Payload, cs) } if err := json.NewEncoder(w).Encode(out); err != nil { From 1483f1e59adcbd91ab90ca01a3b8591e8f37c1fc Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 26 Sep 2020 21:43:49 +0200 Subject: [PATCH 23/80] Add filtering of addresses associated with miners --- cmd/lotus-shed/dealtracker.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 1340dc9c6..719e821a5 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -201,6 +201,26 @@ func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) { return 0, nil } + // Exclude any address associated with a miner + miners, err := dss.api.StateListMiners(ctx, head.Key()) + if err != nil { + log.Warnf("failed to get miner list: %s", err) + return 0, nil + } + for _, m := range miners { + info, err := dss.api.StateMinerInfo(ctx, m, head.Key()) + if err != nil { + log.Warnf("failed to get info for known miner '%s': %s", m, err) + continue + } + + knownFiltered.Store(info.Owner, true) + knownFiltered.Store(info.Worker, true) + for _, a := range info.ControlAddresses { + knownFiltered.Store(a, true) + } + } + deals, err := dss.api.StateMarketDeals(ctx, head.Key()) if err != nil { log.Warnf("failed to get market deals: %s", err) From fb3bcc4ce527961fa172898a180d1fbb865fc133 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 26 Sep 2020 21:49:05 +0200 Subject: [PATCH 24/80] Add startup warning --- cmd/lotus-shed/dealtracker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 719e821a5..8a9d0d6f3 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -303,6 +303,8 @@ var serveDealStatsCmd = &cli.Command{ panic(err) } + log.Warnf("deal-stat server listening on %s\n== NOTE: QUERIES ARE EXPENSIVE - YOU MUST FRONT-CACHE THIS SERVICE\n", list.Addr().String()) + return s.Serve(list) }, } From e4c1f090af73cc0e9f997b3de5e9a9443b493ad5 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sun, 27 Sep 2020 20:44:50 +0200 Subject: [PATCH 25/80] Disable exclusion of miner-associated addresses --- cmd/lotus-shed/dealtracker.go | 40 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 8a9d0d6f3..083db8ecb 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -26,6 +26,8 @@ var resolvedWallets = new(sync.Map) func init() { for _, a := range []string{ "t0100", // client for genesis miner + "t0101", // client for genesis miner + "t0102", // client for genesis miner "t0112", // client for genesis miner "t0113", // client for genesis miner "t0114", // client for genesis miner @@ -201,25 +203,27 @@ func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) { return 0, nil } - // Exclude any address associated with a miner - miners, err := dss.api.StateListMiners(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get miner list: %s", err) - return 0, nil - } - for _, m := range miners { - info, err := dss.api.StateMinerInfo(ctx, m, head.Key()) - if err != nil { - log.Warnf("failed to get info for known miner '%s': %s", m, err) - continue - } + // Disabled as per @pooja's request + // + // // Exclude any address associated with a miner + // miners, err := dss.api.StateListMiners(ctx, head.Key()) + // if err != nil { + // log.Warnf("failed to get miner list: %s", err) + // return 0, nil + // } + // for _, m := range miners { + // info, err := dss.api.StateMinerInfo(ctx, m, head.Key()) + // if err != nil { + // log.Warnf("failed to get info for known miner '%s': %s", m, err) + // continue + // } - knownFiltered.Store(info.Owner, true) - knownFiltered.Store(info.Worker, true) - for _, a := range info.ControlAddresses { - knownFiltered.Store(a, true) - } - } + // knownFiltered.Store(info.Owner, true) + // knownFiltered.Store(info.Worker, true) + // for _, a := range info.ControlAddresses { + // knownFiltered.Store(a, true) + // } + // } deals, err := dss.api.StateMarketDeals(ctx, head.Key()) if err != nil { From be5dc2c57fb8dfde934ddaa63b22d25dd5ca0356 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sun, 27 Sep 2020 20:45:45 +0200 Subject: [PATCH 26/80] Walk back 10 epochs for stat generation --- cmd/lotus-shed/dealtracker.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 083db8ecb..8ded6bf4a 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" lcli "github.com/filecoin-project/lotus/cli" "github.com/ipfs/go-cid" @@ -18,6 +19,10 @@ type dealStatsServer struct { api api.FullNode } +// Requested by @jbenet +// How many epochs back to look at for dealstats +var epochLookback = abi.ChainEpoch(10) + // these lists grow continuously with the network // TODO: need to switch this to an LRU of sorts, to ensure refreshes var knownFiltered = new(sync.Map) @@ -203,6 +208,12 @@ func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) { return 0, nil } + head, err = dss.api.ChainGetTipSetByHeight(ctx, head.Height()-epochLookback, head.Key()) + if err != nil { + log.Warnf("failed to walk back %s epochs: %s", epochLookback, err) + return 0, nil + } + // Disabled as per @pooja's request // // // Exclude any address associated with a miner From e5c56da3217217362cffef257b1433034882c228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 20:10:05 +0100 Subject: [PATCH 27/80] move conformance tvx tool to lotus. --- .github/CODEOWNERS | 1 + cmd/tvx/exec.go | 92 +++++++++ cmd/tvx/extract.go | 380 +++++++++++++++++++++++++++++++++++++ cmd/tvx/main.go | 92 +++++++++ cmd/tvx/state.go | 293 ++++++++++++++++++++++++++++ cmd/tvx/stores.go | 143 ++++++++++++++ conformance/corpus_test.go | 133 +++++++++++++ conformance/driver.go | 8 +- conformance/reporter.go | 62 ++++++ conformance/runner.go | 253 ++++++++++++++++++++++++ conformance/runner_test.go | 376 ------------------------------------ go.sum | 1 + 12 files changed, 1457 insertions(+), 377 deletions(-) create mode 100644 cmd/tvx/exec.go create mode 100644 cmd/tvx/extract.go create mode 100644 cmd/tvx/main.go create mode 100644 cmd/tvx/state.go create mode 100644 cmd/tvx/stores.go create mode 100644 conformance/corpus_test.go create mode 100644 conformance/reporter.go create mode 100644 conformance/runner.go delete mode 100644 conformance/runner_test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 49e461d00..6d717b44d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,3 +13,4 @@ ### Conformance testing. conformance/ @raulk extern/test-vectors @raulk +cmd/tvx @raulk \ No newline at end of file diff --git a/cmd/tvx/exec.go b/cmd/tvx/exec.go new file mode 100644 index 000000000..9ec6f9e2b --- /dev/null +++ b/cmd/tvx/exec.go @@ -0,0 +1,92 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "os" + + "github.com/fatih/color" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/conformance" + + "github.com/filecoin-project/test-vectors/schema" +) + +var execFlags struct { + file string +} + +var execCmd = &cli.Command{ + Name: "exec", + Description: "execute one or many test vectors against Lotus; supplied as a single JSON file, or a ndjson stdin stream", + Action: runExecLotus, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "file", + Usage: "input file; if not supplied, the vector will be read from stdin", + TakesFile: true, + Destination: &execFlags.file, + }, + }, +} + +func runExecLotus(_ *cli.Context) error { + if file := execFlags.file; file != "" { + // we have a single test vector supplied as a file. + file, err := os.Open(file) + if err != nil { + return fmt.Errorf("failed to open test vector: %w", err) + } + + var ( + dec = json.NewDecoder(file) + tv schema.TestVector + ) + + if err = dec.Decode(&tv); err != nil { + return fmt.Errorf("failed to decode test vector: %w", err) + } + + return executeTestVector(tv) + } + + for dec := json.NewDecoder(os.Stdin); ; { + var tv schema.TestVector + switch err := dec.Decode(&tv); err { + case nil: + if err = executeTestVector(tv); err != nil { + return err + } + case io.EOF: + // we're done. + return nil + default: + // something bad happened. + return err + } + } +} + +func executeTestVector(tv schema.TestVector) error { + log.Println("executing test vector:", tv.Meta.ID) + r := new(conformance.LogReporter) + switch class := tv.Class; class { + case "message": + conformance.ExecuteMessageVector(r, &tv) + case "tipset": + conformance.ExecuteTipsetVector(r, &tv) + default: + return fmt.Errorf("test vector class %s not supported", class) + } + + if r.Failed() { + log.Println(color.HiRedString("❌ test vector failed")) + } else { + log.Println(color.GreenString("✅ test vector succeeded")) + } + + return nil +} diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go new file mode 100644 index 000000000..81fd3efbf --- /dev/null +++ b/cmd/tvx/extract.go @@ -0,0 +1,380 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "log" + "os" + + "github.com/filecoin-project/lotus/api" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/conformance" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/test-vectors/schema" + + "github.com/ipfs/go-cid" + "github.com/urfave/cli/v2" +) + +var extractFlags struct { + id string + class string + cid string + file string + retain string +} + +var extractCmd = &cli.Command{ + Name: "extract", + Description: "generate a message-class test vector by extracting it from a live chain", + Action: runExtract, + Flags: []cli.Flag{ + &apiFlag, + &cli.StringFlag{ + Name: "class", + Usage: "class of vector to extract; other required flags depend on the; values: 'message'", + Value: "message", + Destination: &extractFlags.class, + }, + &cli.StringFlag{ + Name: "id", + Usage: "identifier to name this test vector with", + Value: "", + Destination: &extractFlags.id, + }, + &cli.StringFlag{ + Name: "cid", + Usage: "message CID to generate test vector from", + Required: true, + Destination: &extractFlags.cid, + }, + &cli.StringFlag{ + Name: "out", + Aliases: []string{"o"}, + Usage: "file to write test vector to", + Destination: &extractFlags.file, + }, + &cli.StringFlag{ + Name: "state-retain", + Usage: "state retention policy; values: 'accessed-cids', 'accessed-actors'", + Value: "accessed-cids", + Destination: &extractFlags.retain, + }, + }, +} + +func runExtract(_ *cli.Context) error { + // LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering", + // which stashes write operations in a BufferedBlockstore + // (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21) + // such that they're not written until the VM is actually flushed. + // + // For some reason, the standard behaviour was not working for me (raulk), + // and disabling it (such that the state transformations are written immediately + // to the blockstore) worked. + _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") + + ctx := context.Background() + + mcid, err := cid.Decode(extractFlags.cid) + if err != nil { + return err + } + + // Make the API client. + api, closer, err := makeAPIClient() + if err != nil { + return err + } + defer closer() + + log.Printf("locating message with CID: %s", mcid) + + // Locate the message. + msgInfo, err := api.StateSearchMsg(ctx, mcid) + if err != nil { + return fmt.Errorf("failed to locate message: %w", err) + } + + log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) + + // Extract the full message. + msg, err := api.ChainGetMessage(ctx, mcid) + if err != nil { + return err + } + + log.Printf("full message: %+v", msg) + + execTs, incTs, err := fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) + if err != nil { + return err + } + + log.Printf("message was executed in tipset: %s", execTs.Key()) + log.Printf("message was included in tipset: %s", incTs.Key()) + log.Printf("finding precursor messages") + + // Iterate through blocks, finding the one that contains the message and its + // precursors, if any. + var allmsgs []*types.Message + for _, b := range incTs.Blocks() { + messages, err := api.ChainGetBlockMessages(ctx, b.Cid()) + if err != nil { + return err + } + + related, found, err := findMsgAndPrecursors(messages, msg) + if err != nil { + return fmt.Errorf("invariant failed while scanning messages in block %s: %w", b.Cid(), err) + } + + if found { + var mcids []cid.Cid + for _, m := range related { + mcids = append(mcids, m.Cid()) + } + log.Printf("found message in block %s; precursors: %v", b.Cid(), mcids[:len(mcids)-1]) + allmsgs = related + break + } + + log.Printf("message not found in block %s; precursors found: %v; ignoring block", b.Cid(), related) + } + + if allmsgs == nil { + // Message was not found; abort. + return fmt.Errorf("did not find a block containing the message") + } + + precursors := allmsgs[:len(allmsgs)-1] + + var ( + // create a read-through store that uses ChainGetObject to fetch unknown CIDs. + pst = NewProxyingStores(ctx, api) + g = NewSurgeon(ctx, api, pst) + ) + + driver := conformance.NewDriver(ctx, schema.Selector{}) + + // this is the root of the state tree we start with. + root := incTs.ParentState() + log.Printf("base state tree root CID: %s", root) + + // on top of that state tree, we apply all precursors. + log.Printf("number of precursors to apply: %d", len(precursors)) + for i, m := range precursors { + log.Printf("applying precursor %d, cid: %s", i, m.Cid()) + _, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m) + if err != nil { + return fmt.Errorf("failed to execute precursor message: %w", err) + } + } + + var ( + preroot cid.Cid + postroot cid.Cid + applyret *vm.ApplyRet + carWriter func(w io.Writer) error + retention = extractFlags.retain + ) + + log.Printf("using state retention strategy: %s", retention) + switch retention { + case "accessed-cids": + tbs, ok := pst.Blockstore.(TracingBlockstore) + if !ok { + return fmt.Errorf("requested 'accessed-cids' state retention, but no tracing blockstore was present") + } + + tbs.StartTracing() + + preroot = root + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg) + if err != nil { + return fmt.Errorf("failed to execute message: %w", err) + } + accessed := tbs.FinishTracing() + carWriter = func(w io.Writer) error { + return g.WriteCARIncluding(w, accessed, preroot, postroot) + } + + case "accessed-actors": + log.Printf("calculating accessed actors") + // get actors accessed by message. + retain, err := g.GetAccessedActors(ctx, api, mcid) + if err != nil { + return fmt.Errorf("failed to calculate accessed actors: %w", err) + } + // also append the reward actor and the burnt funds actor. + retain = append(retain, reward.Address, builtin.BurntFundsActorAddr, init_.Address) + log.Printf("calculated accessed actors: %v", retain) + + // get the masked state tree from the root, + preroot, err = g.GetMaskedStateTree(root, retain) + if err != nil { + return err + } + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg) + if err != nil { + return fmt.Errorf("failed to execute message: %w", err) + } + carWriter = func(w io.Writer) error { + return g.WriteCAR(w, preroot, postroot) + } + + default: + return fmt.Errorf("unknown state retention option: %s", retention) + } + + msgBytes, err := msg.Serialize() + if err != nil { + return err + } + + var ( + out = new(bytes.Buffer) + gw = gzip.NewWriter(out) + ) + if err := carWriter(gw); err != nil { + return err + } + if err = gw.Flush(); err != nil { + return err + } + if err = gw.Close(); err != nil { + return err + } + + version, err := api.Version(ctx) + if err != nil { + return err + } + + ntwkName, err := api.StateNetworkName(ctx) + if err != nil { + return err + } + + // Write out the test vector. + vector := schema.TestVector{ + Class: schema.ClassMessage, + Meta: &schema.Metadata{ + ID: extractFlags.id, + Gen: []schema.GenerationData{ + {Source: fmt.Sprintf("message:%s:%s", ntwkName, msg.Cid().String())}, + {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, + }, + CAR: out.Bytes(), + Pre: &schema.Preconditions{ + Epoch: int64(execTs.Height()), + StateTree: &schema.StateTree{ + RootCID: preroot, + }, + }, + ApplyMessages: []schema.Message{{Bytes: msgBytes}}, + Post: &schema.Postconditions{ + StateTree: &schema.StateTree{ + RootCID: postroot, + }, + Receipts: []*schema.Receipt{ + { + ExitCode: int64(applyret.ExitCode), + ReturnValue: applyret.Return, + GasUsed: applyret.GasUsed, + }, + }, + }, + } + + output := io.WriteCloser(os.Stdout) + if extractFlags.file != "" { + output, err = os.Create(extractFlags.file) + if err != nil { + return err + } + defer output.Close() + } + + enc := json.NewEncoder(output) + enc.SetIndent("", " ") + if err := enc.Encode(&vector); err != nil { + return err + } + + return nil +} + +// fetchThisAndPrevTipset returns the full tipset identified by the key, as well +// as the previous tipset. In the context of vector generation, the target +// tipset is the one where a message was executed, and the previous tipset is +// the one where the message was included. +func fetchThisAndPrevTipset(ctx context.Context, api api.FullNode, target types.TipSetKey) (targetTs *types.TipSet, prevTs *types.TipSet, err error) { + // get the tipset on which this message was "executed" on. + // https://github.com/filecoin-project/lotus/issues/2847 + targetTs, err = api.ChainGetTipSet(ctx, target) + if err != nil { + return nil, nil, err + } + // get the previous tipset, on which this message was mined, + // i.e. included on-chain. + prevTs, err = api.ChainGetTipSet(ctx, targetTs.Parents()) + if err != nil { + return nil, nil, err + } + return targetTs, prevTs, nil +} + +// findMsgAndPrecursors scans the messages in a block to locate the supplied +// message, looking into the BLS or SECP section depending on the sender's +// address type. +// +// It returns any precursors (if they exist), and the found message (if found), +// in a slice. +// +// It also returns a boolean indicating whether the message was actually found. +// +// This function also asserts invariants, and if those fail, it returns an error. +func findMsgAndPrecursors(messages *api.BlockMessages, target *types.Message) (related []*types.Message, found bool, err error) { + // Decide which block of messages to process, depending on whether the + // sender is a BLS or a SECP account. + input := messages.BlsMessages + if senderKind := target.From.Protocol(); senderKind == address.SECP256K1 { + input = make([]*types.Message, 0, len(messages.SecpkMessages)) + for _, sm := range messages.SecpkMessages { + input = append(input, &sm.Message) + } + } + + for _, other := range input { + if other.From != target.From { + continue + } + + // this message is from the same sender, so it's related. + related = append(related, other) + + if other.Nonce > target.Nonce { + return nil, false, fmt.Errorf("a message with nonce higher than the target was found before the target; offending mcid: %s", other.Cid()) + } + + // this message is the target; we're done. + if other.Cid() == target.Cid() { + return related, true, nil + } + } + + // this could happen because a block contained related messages, but not + // the target (that is, messages with a lower nonce, but ultimately not the + // target). + return related, false, nil +} diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go new file mode 100644 index 000000000..183c2fbe1 --- /dev/null +++ b/cmd/tvx/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "sort" + "strings" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" +) + +var apiEndpoint string + +var apiFlag = cli.StringFlag{ + Name: "api", + Usage: "json-rpc api endpoint, formatted as token:multiaddr", + EnvVars: []string{"FULLNODE_API_INFO"}, + DefaultText: "", + Destination: &apiEndpoint, +} + +func main() { + app := &cli.App{ + Name: "tvx", + Description: `tvx is a tool for extracting and executing test vectors. It has two subcommands. + + tvx extract extracts a test vector from a live network. It requires access to + a Filecoin client that exposes the standard JSON-RPC API endpoint. Set the API + endpoint on the FULLNODE_API_INFO env variable, or through the --api flag. The + format is token:multiaddr. Only message class test vectors are supported + for now. + + tvx exec executes test vectors against Lotus. Either you can supply one in a + file, or many as an ndjson stdin stream.`, + Usage: "tvx is a tool for extracting and executing test vectors", + Commands: []*cli.Command{ + extractCmd, + execCmd, + }, + } + + sort.Sort(cli.CommandsByName(app.Commands)) + for _, c := range app.Commands { + sort.Sort(cli.FlagsByName(c.Flags)) + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} + +func makeAPIClient() (api.FullNode, jsonrpc.ClientCloser, error) { + sp := strings.SplitN(apiEndpoint, ":", 2) + if len(sp) != 2 { + return nil, nil, fmt.Errorf("invalid api value, missing token or address: %s", apiEndpoint) + } + + token := sp[0] + ma, err := multiaddr.NewMultiaddr(sp[1]) + if err != nil { + return nil, nil, fmt.Errorf("could not parse provided multiaddr: %w", err) + } + + _, dialAddr, err := manet.DialArgs(ma) + if err != nil { + return nil, nil, fmt.Errorf("invalid api multiAddr: %w", err) + } + + var ( + addr = "ws://" + dialAddr + "/rpc/v0" + headers = make(http.Header, 1) + ) + if len(token) != 0 { + headers.Add("Authorization", "Bearer "+token) + } + + node, closer, err := client.NewFullNodeRPC(context.Background(), addr, headers) + if err != nil { + return nil, nil, fmt.Errorf("could not connect to api: %w", err) + } + return node, closer, nil +} diff --git a/cmd/tvx/state.go b/cmd/tvx/state.go new file mode 100644 index 000000000..cef5c5494 --- /dev/null +++ b/cmd/tvx/state.go @@ -0,0 +1,293 @@ +package main + +import ( + "context" + "fmt" + "io" + "log" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipld-format" + "github.com/ipld/go-car" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lotus/api" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/state" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" +) + +// StateSurgeon is an object used to fetch and manipulate state. +type StateSurgeon struct { + ctx context.Context + api api.FullNode + stores *Stores +} + +// NewSurgeon returns a state surgeon, an object used to fetch and manipulate +// state. +func NewSurgeon(ctx context.Context, api api.FullNode, stores *Stores) *StateSurgeon { + return &StateSurgeon{ + ctx: ctx, + api: api, + stores: stores, + } +} + +// GetMaskedStateTree trims the state tree at the supplied tipset to contain +// only the state of the actors in the retain set. It also "dives" into some +// singleton system actors, like the init actor, to trim the state so as to +// compute a minimal state tree. In the future, thid method will dive into +// other system actors like the power actor and the market actor. +func (sg *StateSurgeon) GetMaskedStateTree(previousRoot cid.Cid, retain []address.Address) (cid.Cid, error) { + // TODO: this will need to be parameterized on network version. + st, err := state.LoadStateTree(sg.stores.CBORStore, previousRoot) + if err != nil { + return cid.Undef, err + } + + initActor, initState, err := sg.loadInitActor(st) + if err != nil { + return cid.Undef, err + } + + err = sg.retainInitEntries(initState, retain) + if err != nil { + return cid.Undef, err + } + + err = sg.saveInitActor(initActor, initState, st) + if err != nil { + return cid.Undef, err + } + + // resolve all addresses to ID addresses. + resolved, err := sg.resolveAddresses(retain, initState) + if err != nil { + return cid.Undef, err + } + + st, err = sg.transplantActors(st, resolved) + if err != nil { + return cid.Undef, err + } + + root, err := st.Flush(sg.ctx) + if err != nil { + return cid.Undef, err + } + + return root, nil +} + +// GetAccessedActors identifies the actors that were accessed during the +// execution of a message. +func (sg *StateSurgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid cid.Cid) ([]address.Address, error) { + log.Printf("calculating accessed actors during execution of message: %s", mid) + msgInfo, err := a.StateSearchMsg(ctx, mid) + if err != nil { + return nil, err + } + if msgInfo == nil { + return nil, fmt.Errorf("message info is nil") + } + + msgObj, err := a.ChainGetMessage(ctx, mid) + if err != nil { + return nil, err + } + + ts, err := a.ChainGetTipSet(ctx, msgInfo.TipSet) + if err != nil { + return nil, err + } + + trace, err := a.StateCall(ctx, msgObj, ts.Parents()) + if err != nil { + return nil, fmt.Errorf("could not replay msg: %w", err) + } + + accessed := make(map[address.Address]struct{}) + + var recur func(trace *types.ExecutionTrace) + recur = func(trace *types.ExecutionTrace) { + accessed[trace.Msg.To] = struct{}{} + accessed[trace.Msg.From] = struct{}{} + for _, s := range trace.Subcalls { + recur(&s) + } + } + recur(&trace.ExecutionTrace) + + ret := make([]address.Address, 0, len(accessed)) + for k := range accessed { + ret = append(ret, k) + } + + return ret, nil +} + +// WriteCAR recursively writes the tree referenced by the root as a CAR into the +// supplied io.Writer. +func (sg *StateSurgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error { + carWalkFn := func(nd format.Node) (out []*format.Link, err error) { + for _, link := range nd.Links() { + if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { + continue + } + out = append(out, link) + } + return out, nil + } + return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn) +} + +// WriteCARIncluding writes a CAR including only the CIDs that are listed in +// the include set. This leads to an intentially sparse tree with dangling links. +func (sg *StateSurgeon) WriteCARIncluding(w io.Writer, include map[cid.Cid]struct{}, roots ...cid.Cid) error { + carWalkFn := func(nd format.Node) (out []*format.Link, err error) { + for _, link := range nd.Links() { + if _, ok := include[link.Cid]; !ok { + continue + } + if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { + continue + } + out = append(out, link) + } + return out, nil + } + return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn) +} + +// transplantActors plucks the state from the supplied actors at the given +// tipset, and places it into the supplied state map. +func (sg *StateSurgeon) transplantActors(src *state.StateTree, pluck []address.Address) (*state.StateTree, error) { + log.Printf("transplanting actor states: %v", pluck) + + dst, err := state.NewStateTree(sg.stores.CBORStore, src.Version()) + if err != nil { + return nil, err + } + + for _, a := range pluck { + actor, err := src.GetActor(a) + if err != nil { + return nil, fmt.Errorf("get actor %s failed: %w", a, err) + } + + err = dst.SetActor(a, actor) + if err != nil { + return nil, err + } + + // recursive copy of the actor state. + err = vm.Copy(context.TODO(), sg.stores.Blockstore, sg.stores.Blockstore, actor.Head) + if err != nil { + return nil, err + } + + actorState, err := sg.api.ChainReadObj(sg.ctx, actor.Head) + if err != nil { + return nil, err + } + + cid, err := sg.stores.CBORStore.Put(sg.ctx, &cbg.Deferred{Raw: actorState}) + if err != nil { + return nil, err + } + + if cid != actor.Head { + panic("mismatched cids") + } + } + + return dst, nil +} + +// saveInitActor saves the state of the init actor to the provided state map. +func (sg *StateSurgeon) saveInitActor(initActor *types.Actor, initState init_.State, st *state.StateTree) error { + log.Printf("saving init actor into state tree") + + // Store the state of the init actor. + cid, err := sg.stores.CBORStore.Put(sg.ctx, initState) + if err != nil { + return err + } + actor := *initActor + actor.Head = cid + + err = st.SetActor(init_.Address, &actor) + if err != nil { + return err + } + + cid, _ = st.Flush(sg.ctx) + log.Printf("saved init actor into state tree; new root: %s", cid) + return nil +} + +// retainInitEntries takes an old init actor state, and retains only the +// entries in the retain set, returning a new init actor state. +func (sg *StateSurgeon) retainInitEntries(state init_.State, retain []address.Address) error { + log.Printf("retaining init actor entries for addresses: %v", retain) + + m := make(map[address.Address]struct{}, len(retain)) + for _, a := range retain { + m[a] = struct{}{} + } + + var remove []address.Address + _ = state.ForEachActor(func(id abi.ActorID, address address.Address) error { + if _, ok := m[address]; !ok { + remove = append(remove, address) + } + return nil + }) + + err := state.Remove(remove...) + log.Printf("new init actor state: %+v", state) + return err +} + +// resolveAddresses resolved the requested addresses from the provided +// InitActor state, returning a slice of length len(orig), where each index +// contains the resolved address. +func (sg *StateSurgeon) resolveAddresses(orig []address.Address, ist init_.State) (ret []address.Address, err error) { + log.Printf("resolving addresses: %v", orig) + + ret = make([]address.Address, len(orig)) + for i, addr := range orig { + resolved, found, err := ist.ResolveAddress(addr) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("address not found: %s", addr) + } + ret[i] = resolved + } + + log.Printf("resolved addresses: %v", ret) + return ret, nil +} + +// loadInitActor loads the init actor state from a given tipset. +func (sg *StateSurgeon) loadInitActor(st *state.StateTree) (*types.Actor, init_.State, error) { + actor, err := st.GetActor(init_.Address) + if err != nil { + return nil, nil, err + } + + initState, err := init_.Load(sg.stores.ADTStore, actor) + if err != nil { + return nil, nil, err + } + + log.Printf("loaded init actor state: %+v", initState) + + return actor, initState, nil +} diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go new file mode 100644 index 000000000..7d3fd5e3a --- /dev/null +++ b/cmd/tvx/stores.go @@ -0,0 +1,143 @@ +package main + +import ( + "context" + "log" + "sync" + + "github.com/fatih/color" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/lib/blockstore" + + "github.com/filecoin-project/specs-actors/actors/util/adt" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + exchange "github.com/ipfs/go-ipfs-exchange-interface" + offline "github.com/ipfs/go-ipfs-exchange-offline" + cbor "github.com/ipfs/go-ipld-cbor" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" +) + +// Stores is a collection of the different stores and services that are needed +// to deal with the data layer of Filecoin, conveniently interlinked with one +// another. +type Stores struct { + CBORStore cbor.IpldStore + ADTStore adt.Store + Datastore ds.Batching + Blockstore blockstore.Blockstore + BlockService blockservice.BlockService + Exchange exchange.Interface + DAGService format.DAGService +} + +// NewProxyingStores is a set of Stores backed by a proxying Blockstore that +// proxies Get requests for unknown CIDs to a Filecoin node, via the +// ChainReadObj RPC. +func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores { + ds := ds.NewMapDatastore() + + bs := &proxyingBlockstore{ + ctx: ctx, + api: api, + Blockstore: blockstore.NewBlockstore(ds), + } + + return NewStores(ctx, ds, bs) +} + +// NewStores creates a non-proxying set of Stores. +func NewStores(ctx context.Context, ds ds.Batching, bs blockstore.Blockstore) *Stores { + var ( + cborstore = cbor.NewCborStore(bs) + offl = offline.Exchange(bs) + blkserv = blockservice.New(bs, offl) + dserv = merkledag.NewDAGService(blkserv) + ) + + return &Stores{ + CBORStore: cborstore, + ADTStore: adt.WrapStore(ctx, cborstore), + Datastore: ds, + Blockstore: bs, + Exchange: offl, + BlockService: blkserv, + DAGService: dserv, + } +} + +// TracingBlockstore is a Blockstore trait that records CIDs that were accessed +// through Get. +type TracingBlockstore interface { + // StartTracing starts tracing CIDs accessed through the this Blockstore. + StartTracing() + + // FinishTracing finishes tracing accessed CIDs, and returns a map of the + // CIDs that were traced. + FinishTracing() map[cid.Cid]struct{} +} + +// proxyingBlockstore is a Blockstore wrapper that fetches unknown CIDs from +// a Filecoin node via JSON-RPC. +type proxyingBlockstore struct { + ctx context.Context + api api.FullNode + + lk sync.RWMutex + tracing bool + traced map[cid.Cid]struct{} + + blockstore.Blockstore +} + +var _ TracingBlockstore = (*proxyingBlockstore)(nil) + +func (pb *proxyingBlockstore) StartTracing() { + pb.lk.Lock() + pb.tracing = true + pb.traced = map[cid.Cid]struct{}{} + pb.lk.Unlock() +} + +func (pb *proxyingBlockstore) FinishTracing() map[cid.Cid]struct{} { + pb.lk.Lock() + ret := pb.traced + pb.tracing = false + pb.traced = map[cid.Cid]struct{}{} + pb.lk.Unlock() + return ret +} + +func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { + pb.lk.RLock() + if pb.tracing { + pb.traced[cid] = struct{}{} + } + pb.lk.RUnlock() + + if block, err := pb.Blockstore.Get(cid); err == nil { + return block, err + } + + log.Println(color.CyanString("fetching cid via rpc: %v", cid)) + item, err := pb.api.ChainReadObj(pb.ctx, cid) + if err != nil { + return nil, err + } + block, err := blocks.NewBlockWithCid(item, cid) + if err != nil { + return nil, err + } + + err = pb.Blockstore.Put(block) + if err != nil { + return nil, err + } + + return block, nil +} diff --git a/conformance/corpus_test.go b/conformance/corpus_test.go new file mode 100644 index 000000000..3d447570d --- /dev/null +++ b/conformance/corpus_test.go @@ -0,0 +1,133 @@ +package conformance + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/filecoin-project/test-vectors/schema" +) + +const ( + // EnvSkipConformance, if 1, skips the conformance test suite. + EnvSkipConformance = "SKIP_CONFORMANCE" + + // EnvCorpusRootDir is the name of the environment variable where the path + // to an alternative corpus location can be provided. + // + // The default is defaultCorpusRoot. + EnvCorpusRootDir = "CORPUS_DIR" + + // defaultCorpusRoot is the directory where the test vector corpus is hosted. + // It is mounted on the Lotus repo as a git submodule. + // + // When running this test, the corpus root can be overridden through the + // -conformance.corpus CLI flag to run an alternate corpus. + defaultCorpusRoot = "../extern/test-vectors/corpus" +) + +// ignore is a set of paths relative to root to skip. +var ignore = map[string]struct{}{ + ".git": {}, + "schema.json": {}, +} + +// TestConformance is the entrypoint test that runs all test vectors found +// in the corpus root directory. +// +// It locates all json files via a recursive walk, skipping over the ignore set, +// as well as files beginning with _. It parses each file as a test vector, and +// runs it via the Driver. +func TestConformance(t *testing.T) { + if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" { + t.SkipNow() + } + // corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag, + // falling back to defaultCorpusRoot if not provided. + corpusRoot := defaultCorpusRoot + if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" { + corpusRoot = dir + } + + var vectors []string + err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error { + if err != nil { + t.Fatal(err) + } + + filename := filepath.Base(path) + rel, err := filepath.Rel(corpusRoot, path) + if err != nil { + t.Fatal(err) + } + + if _, ok := ignore[rel]; ok { + // skip over using the right error. + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + if info.IsDir() { + // dive into directories. + return nil + } + if filepath.Ext(path) != ".json" { + // skip if not .json. + return nil + } + if ignored := strings.HasPrefix(filename, "_"); ignored { + // ignore files starting with _. + t.Logf("ignoring: %s", rel) + return nil + } + vectors = append(vectors, rel) + return nil + }) + + if err != nil { + t.Fatal(err) + } + + if len(vectors) == 0 { + t.Fatalf("no test vectors found") + } + + // Run a test for each vector. + for _, v := range vectors { + path := filepath.Join(corpusRoot, v) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("failed to read test raw file: %s", path) + } + + var vector schema.TestVector + err = json.Unmarshal(raw, &vector) + if err != nil { + t.Errorf("failed to parse test vector %s: %s; skipping", path, err) + continue + } + + t.Run(v, func(t *testing.T) { + for _, h := range vector.Hints { + if h == schema.HintIncorrect { + t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID) + t.SkipNow() + } + } + + // dispatch the execution depending on the vector class. + switch vector.Class { + case "message": + ExecuteMessageVector(t, &vector) + case "tipset": + ExecuteTipsetVector(t, &vector) + default: + t.Fatalf("test vector class not supported: %s", vector.Class) + } + }) + } +} diff --git a/conformance/driver.go b/conformance/driver.go index f43a8739d..ee9727cae 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -5,6 +5,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -154,7 +155,12 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch return nil, cid.Undef, err } - root, err := lvm.Flush(d.ctx) + // do not flush the VM, as this forces a recursive copy to the blockstore, + // walking the full state tree, which we don't require. + // root, err := lvm.Flush(d.ctx) + // + // instead, flush the pending writes on the state tree. + root, err := lvm.StateTree().(*state.StateTree).Flush(d.ctx) return ret, root, err } diff --git a/conformance/reporter.go b/conformance/reporter.go new file mode 100644 index 000000000..747caae32 --- /dev/null +++ b/conformance/reporter.go @@ -0,0 +1,62 @@ +package conformance + +import ( + "log" + "os" + "sync/atomic" + "testing" + + "github.com/fatih/color" +) + +// Reporter is a contains a subset of the testing.T methods, so that the +// Execute* functions in this package can be used inside or outside of +// go test runs. +type Reporter interface { + Helper() + + Log(args ...interface{}) + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) + Logf(format string, args ...interface{}) + FailNow() + Failed() bool +} + +var _ Reporter = (*testing.T)(nil) + +// LogReporter wires the Reporter methods to the log package. It is appropriate +// to use when calling the Execute* functions from a standalone CLI program. +type LogReporter struct { + failed int32 +} + +var _ Reporter = (*LogReporter)(nil) + +func (_ *LogReporter) Helper() {} + +func (_ *LogReporter) Log(args ...interface{}) { + log.Println(args...) +} + +func (_ *LogReporter) Logf(format string, args ...interface{}) { + log.Printf(format, args...) +} + +func (_ *LogReporter) FailNow() { + os.Exit(1) +} + +func (l *LogReporter) Failed() bool { + return atomic.LoadInt32(&l.failed) == 1 +} + +func (l *LogReporter) Errorf(format string, args ...interface{}) { + atomic.StoreInt32(&l.failed, 1) + log.Println(color.HiRedString("❌ "+format, args...)) +} + +func (l *LogReporter) Fatalf(format string, args ...interface{}) { + atomic.StoreInt32(&l.failed, 1) + log.Fatal(color.HiRedString("❌ "+format, args...)) +} diff --git a/conformance/runner.go b/conformance/runner.go new file mode 100644 index 000000000..1fc1c1425 --- /dev/null +++ b/conformance/runner.go @@ -0,0 +1,253 @@ +package conformance + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strconv" + + "github.com/fatih/color" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/test-vectors/schema" + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/ipld/go-car" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" +) + +// ExecuteMessageVector executes a message-class test vector. +func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { + var ( + ctx = context.Background() + epoch = vector.Pre.Epoch + root = vector.Pre.StateTree.RootCID + ) + + // Load the CAR into a new temporary Blockstore. + bs, err := LoadVectorCAR(vector.CAR) + if err != nil { + r.Fatalf("failed to load the vector CAR: %w", err) + } + + // Create a new Driver. + driver := NewDriver(ctx, vector.Selector) + + // Apply every message. + for i, m := range vector.ApplyMessages { + msg, err := types.DecodeMessage(m.Bytes) + if err != nil { + r.Fatalf("failed to deserialize message: %s", err) + } + + // add an epoch if one's set. + if m.Epoch != nil { + epoch = *m.Epoch + } + + // Execute the message. + var ret *vm.ApplyRet + ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg) + if err != nil { + r.Fatalf("fatal failure when executing message: %s", err) + } + + // Assert that the receipt matches what the test vector expects. + assertMsgResult(r, vector.Post.Receipts[i], ret, strconv.Itoa(i)) + } + + // Once all messages are applied, assert that the final state root matches + // the expected postcondition root. + if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual { + r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual) + dumpThreeWayStateDiff(r, vector, bs, root) + r.FailNow() + } +} + +// ExecuteTipsetVector executes a tipset-class test vector. +func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { + var ( + ctx = context.Background() + prevEpoch = vector.Pre.Epoch + root = vector.Pre.StateTree.RootCID + tmpds = ds.NewMapDatastore() + ) + + // Load the vector CAR into a new temporary Blockstore. + bs, err := LoadVectorCAR(vector.CAR) + if err != nil { + r.Fatalf("failed to load the vector CAR: %w", err) + } + + // Create a new Driver. + driver := NewDriver(ctx, vector.Selector) + + // Apply every tipset. + var receiptsIdx int + for i, ts := range vector.ApplyTipsets { + ts := ts // capture + ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts) + if err != nil { + r.Fatalf("failed to apply tipset %d message: %s", i, err) + } + + for j, v := range ret.AppliedResults { + assertMsgResult(r, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i)) + receiptsIdx++ + } + + // Compare the receipts root. + if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual { + r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual) + } + + prevEpoch = ts.Epoch + root = ret.PostStateRoot + } + + // Once all messages are applied, assert that the final state root matches + // the expected postcondition root. + if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual { + r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual) + dumpThreeWayStateDiff(r, vector, bs, root) + r.FailNow() + } +} + +// assertMsgResult compares a message result. It takes the expected receipt +// encoded in the vector, the actual receipt returned by Lotus, and a message +// label to log in the assertion failure message to facilitate debugging. +func assertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet, label string) { + r.Helper() + + if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual { + r.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual) + } + if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual { + r.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual) + } + if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) { + r.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual)) + } +} + +func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) { + // check if statediff exists; if not, skip. + if err := exec.Command("statediff", "--help").Run(); err != nil { + r.Log("could not dump 3-way state tree diff upon test failure: statediff command not found") + r.Log("install statediff with:") + r.Log("$ git clone https://github.com/filecoin-project/statediff.git") + r.Log("$ cd statediff") + r.Log("$ go generate ./...") + r.Log("$ go install ./cmd/statediff") + return + } + + tmpCar, err := writeStateToTempCAR(bs, + vector.Pre.StateTree.RootCID, + vector.Post.StateTree.RootCID, + actual, + ) + if err != nil { + r.Fatalf("failed to write temporary state CAR: %s", err) + } + defer os.RemoveAll(tmpCar) + + color.NoColor = false // enable colouring. + + var ( + a = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state") + b = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state") + c = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state") + d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]") + d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]") + d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]") + ) + + printDiff := func(left, right cid.Cid) { + cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String()) + b, err := cmd.CombinedOutput() + if err != nil { + r.Fatalf("statediff failed: %s", err) + } + r.Log(string(b)) + } + + bold := color.New(color.Bold).SprintfFunc() + + // run state diffs. + r.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c)) + + r.Log(bold("--- %s left: %s; right: %s ---", d1, a, b)) + printDiff(vector.Post.StateTree.RootCID, actual) + + r.Log(bold("--- %s left: %s; right: %s ---", d2, c, b)) + printDiff(vector.Pre.StateTree.RootCID, actual) + + r.Log(bold("--- %s left: %s; right: %s ---", d3, c, a)) + printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID) +} + +// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be +// cleaned up via t.Cleanup(). It returns the full path of the temp file. +func writeStateToTempCAR(bs blockstore.Blockstore, roots ...cid.Cid) (string, error) { + tmp, err := ioutil.TempFile("", "lotus-tests-*.car") + if err != nil { + return "", fmt.Errorf("failed to create temp file to dump CAR for diffing: %w", err) + } + + carWalkFn := func(nd format.Node) (out []*format.Link, err error) { + for _, link := range nd.Links() { + if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { + continue + } + out = append(out, link) + } + return out, nil + } + + var ( + offl = offline.Exchange(bs) + blkserv = blockservice.New(bs, offl) + dserv = merkledag.NewDAGService(blkserv) + ) + + err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn) + if err != nil { + return "", fmt.Errorf("failed to dump CAR for diffing: %w", err) + } + _ = tmp.Close() + return tmp.Name(), nil +} + +func LoadVectorCAR(vectorCAR schema.Base64EncodedBytes) (blockstore.Blockstore, error) { + bs := blockstore.NewTemporary() + + // Read the base64-encoded CAR from the vector, and inflate the gzip. + buf := bytes.NewReader(vectorCAR) + r, err := gzip.NewReader(buf) + if err != nil { + return nil, fmt.Errorf("failed to inflate gzipped CAR: %s", err) + } + defer r.Close() // nolint + + // Load the CAR embedded in the test vector into the Blockstore. + _, err = car.LoadCar(bs, r) + if err != nil { + return nil, fmt.Errorf("failed to load state tree car from test vector: %s", err) + } + return bs, nil +} diff --git a/conformance/runner_test.go b/conformance/runner_test.go deleted file mode 100644 index cc7ef6b3d..000000000 --- a/conformance/runner_test.go +++ /dev/null @@ -1,376 +0,0 @@ -package conformance - -import ( - "bytes" - "compress/gzip" - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "testing" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/exitcode" - "github.com/ipfs/go-blockservice" - "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - offline "github.com/ipfs/go-ipfs-exchange-offline" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/lib/blockstore" - - "github.com/filecoin-project/test-vectors/schema" - - "github.com/fatih/color" - "github.com/ipld/go-car" -) - -const ( - // EnvSkipConformance, if 1, skips the conformance test suite. - EnvSkipConformance = "SKIP_CONFORMANCE" - - // EnvCorpusRootDir is the name of the environment variable where the path - // to an alternative corpus location can be provided. - // - // The default is defaultCorpusRoot. - EnvCorpusRootDir = "CORPUS_DIR" - - // defaultCorpusRoot is the directory where the test vector corpus is hosted. - // It is mounted on the Lotus repo as a git submodule. - // - // When running this test, the corpus root can be overridden through the - // -conformance.corpus CLI flag to run an alternate corpus. - defaultCorpusRoot = "../extern/test-vectors/corpus" -) - -// ignore is a set of paths relative to root to skip. -var ignore = map[string]struct{}{ - ".git": {}, - "schema.json": {}, -} - -// TestConformance is the entrypoint test that runs all test vectors found -// in the corpus root directory. -// -// It locates all json files via a recursive walk, skipping over the ignore set, -// as well as files beginning with _. It parses each file as a test vector, and -// runs it via the Driver. -func TestConformance(t *testing.T) { - if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" { - t.SkipNow() - } - // corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag, - // falling back to defaultCorpusRoot if not provided. - corpusRoot := defaultCorpusRoot - if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" { - corpusRoot = dir - } - - var vectors []string - err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error { - if err != nil { - t.Fatal(err) - } - - filename := filepath.Base(path) - rel, err := filepath.Rel(corpusRoot, path) - if err != nil { - t.Fatal(err) - } - - if _, ok := ignore[rel]; ok { - // skip over using the right error. - if info.IsDir() { - return filepath.SkipDir - } - return nil - } - if info.IsDir() { - // dive into directories. - return nil - } - if filepath.Ext(path) != ".json" { - // skip if not .json. - return nil - } - if ignored := strings.HasPrefix(filename, "_"); ignored { - // ignore files starting with _. - t.Logf("ignoring: %s", rel) - return nil - } - vectors = append(vectors, rel) - return nil - }) - - if err != nil { - t.Fatal(err) - } - - if len(vectors) == 0 { - t.Fatalf("no test vectors found") - } - - // Run a test for each vector. - for _, v := range vectors { - path := filepath.Join(corpusRoot, v) - raw, err := ioutil.ReadFile(path) - if err != nil { - t.Fatalf("failed to read test raw file: %s", path) - } - - var vector schema.TestVector - err = json.Unmarshal(raw, &vector) - if err != nil { - t.Errorf("failed to parse test vector %s: %s; skipping", path, err) - continue - } - - t.Run(v, func(t *testing.T) { - for _, h := range vector.Hints { - if h == schema.HintIncorrect { - t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID) - t.SkipNow() - } - } - - // dispatch the execution depending on the vector class. - switch vector.Class { - case "message": - executeMessageVector(t, &vector) - case "tipset": - executeTipsetVector(t, &vector) - default: - t.Fatalf("test vector class not supported: %s", vector.Class) - } - }) - } -} - -// executeMessageVector executes a message-class test vector. -func executeMessageVector(t *testing.T, vector *schema.TestVector) { - var ( - ctx = context.Background() - epoch = vector.Pre.Epoch - root = vector.Pre.StateTree.RootCID - ) - - // Load the CAR into a new temporary Blockstore. - bs := loadCAR(t, vector.CAR) - - // Create a new Driver. - driver := NewDriver(ctx, vector.Selector) - - // Apply every message. - for i, m := range vector.ApplyMessages { - msg, err := types.DecodeMessage(m.Bytes) - if err != nil { - t.Fatalf("failed to deserialize message: %s", err) - } - - // add an epoch if one's set. - if m.Epoch != nil { - epoch = *m.Epoch - } - - // Execute the message. - var ret *vm.ApplyRet - ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg) - if err != nil { - t.Fatalf("fatal failure when executing message: %s", err) - } - - // Assert that the receipt matches what the test vector expects. - assertMsgResult(t, vector.Post.Receipts[i], ret, strconv.Itoa(i)) - } - - // Once all messages are applied, assert that the final state root matches - // the expected postcondition root. - if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual { - t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual) - dumpThreeWayStateDiff(t, vector, bs, root) - t.FailNow() - } -} - -// executeTipsetVector executes a tipset-class test vector. -func executeTipsetVector(t *testing.T, vector *schema.TestVector) { - var ( - ctx = context.Background() - prevEpoch = vector.Pre.Epoch - root = vector.Pre.StateTree.RootCID - tmpds = ds.NewMapDatastore() - ) - - // Load the CAR into a new temporary Blockstore. - bs := loadCAR(t, vector.CAR) - - // Create a new Driver. - driver := NewDriver(ctx, vector.Selector) - - // Apply every tipset. - var receiptsIdx int - for i, ts := range vector.ApplyTipsets { - ts := ts // capture - ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts) - if err != nil { - t.Fatalf("failed to apply tipset %d message: %s", i, err) - } - - for j, v := range ret.AppliedResults { - assertMsgResult(t, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i)) - receiptsIdx++ - } - - // Compare the receipts root. - if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual { - t.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual) - } - - prevEpoch = ts.Epoch - root = ret.PostStateRoot - } - - // Once all messages are applied, assert that the final state root matches - // the expected postcondition root. - if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual { - t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual) - dumpThreeWayStateDiff(t, vector, bs, root) - t.FailNow() - } -} - -// assertMsgResult compares a message result. It takes the expected receipt -// encoded in the vector, the actual receipt returned by Lotus, and a message -// label to log in the assertion failure message to facilitate debugging. -func assertMsgResult(t *testing.T, expected *schema.Receipt, actual *vm.ApplyRet, label string) { - t.Helper() - - if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual { - t.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual) - } - if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual { - t.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual) - } - if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) { - t.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual)) - } -} - -func dumpThreeWayStateDiff(t *testing.T, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) { - // check if statediff exists; if not, skip. - if err := exec.Command("statediff", "--help").Run(); err != nil { - t.Log("could not dump 3-way state tree diff upon test failure: statediff command not found") - t.Log("install statediff with:") - t.Log("$ git clone https://github.com/filecoin-project/statediff.git") - t.Log("$ cd statediff") - t.Log("$ go generate ./...") - t.Log("$ go install ./cmd/statediff") - return - } - - tmpCar := writeStateToTempCAR(t, bs, - vector.Pre.StateTree.RootCID, - vector.Post.StateTree.RootCID, - actual, - ) - - color.NoColor = false // enable colouring. - - t.Errorf("wrong post root cid; expected %v, but got %v", vector.Post.StateTree.RootCID, actual) - - var ( - a = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state") - b = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state") - c = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state") - d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]") - d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]") - d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]") - ) - - printDiff := func(left, right cid.Cid) { - cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String()) - b, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("statediff failed: %s", err) - } - t.Log(string(b)) - } - - bold := color.New(color.Bold).SprintfFunc() - - // run state diffs. - t.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c)) - - t.Log(bold("--- %s left: %s; right: %s ---", d1, a, b)) - printDiff(vector.Post.StateTree.RootCID, actual) - - t.Log(bold("--- %s left: %s; right: %s ---", d2, c, b)) - printDiff(vector.Pre.StateTree.RootCID, actual) - - t.Log(bold("--- %s left: %s; right: %s ---", d3, c, a)) - printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID) -} - -// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be -// cleaned up via t.Cleanup(). It returns the full path of the temp file. -func writeStateToTempCAR(t *testing.T, bs blockstore.Blockstore, roots ...cid.Cid) string { - tmp, err := ioutil.TempFile("", "lotus-tests-*.car") - if err != nil { - t.Fatalf("failed to create temp file to dump CAR for diffing: %s", err) - } - // register a cleanup function to delete the CAR. - t.Cleanup(func() { - _ = os.Remove(tmp.Name()) - }) - - carWalkFn := func(nd format.Node) (out []*format.Link, err error) { - for _, link := range nd.Links() { - if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { - continue - } - out = append(out, link) - } - return out, nil - } - - var ( - offl = offline.Exchange(bs) - blkserv = blockservice.New(bs, offl) - dserv = merkledag.NewDAGService(blkserv) - ) - - err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn) - if err != nil { - t.Fatalf("failed to dump CAR for diffing: %s", err) - } - _ = tmp.Close() - return tmp.Name() -} - -func loadCAR(t *testing.T, vectorCAR schema.Base64EncodedBytes) blockstore.Blockstore { - bs := blockstore.NewTemporary() - - // Read the base64-encoded CAR from the vector, and inflate the gzip. - buf := bytes.NewReader(vectorCAR) - r, err := gzip.NewReader(buf) - if err != nil { - t.Fatalf("failed to inflate gzipped CAR: %s", err) - } - defer r.Close() // nolint - - // Load the CAR embedded in the test vector into the Blockstore. - _, err = car.LoadCar(bs, r) - if err != nil { - t.Fatalf("failed to load state tree car from test vector: %s", err) - } - return bs -} diff --git a/go.sum b/go.sum index 6412fe743..bdb80aa83 100644 --- a/go.sum +++ b/go.sum @@ -504,6 +504,7 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= From a712c109d804c09988d69a741a10b17fd2312f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 20:30:32 +0100 Subject: [PATCH 28/80] tvx/extract: print confirmation. --- cmd/tvx/extract.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 81fd3efbf..62314c6f6 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -48,7 +48,7 @@ var extractCmd = &cli.Command{ &cli.StringFlag{ Name: "id", Usage: "identifier to name this test vector with", - Value: "", + Value: "(undefined)", Destination: &extractFlags.id, }, &cli.StringFlag{ @@ -297,12 +297,13 @@ func runExtract(_ *cli.Context) error { } output := io.WriteCloser(os.Stdout) - if extractFlags.file != "" { - output, err = os.Create(extractFlags.file) + if file := extractFlags.file; file != "" { + output, err = os.Create(file) if err != nil { return err } defer output.Close() + defer log.Printf("wrote test vector to file: %s", file) } enc := json.NewEncoder(output) From f5f23f7291ab83c021e86164d775c896d261ea52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 20:55:09 +0100 Subject: [PATCH 29/80] driver: option for VM flushing. --- cmd/tvx/extract.go | 4 +++- conformance/driver.go | 35 +++++++++++++++++++++++++++-------- conformance/runner.go | 4 ++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 62314c6f6..06b3d6fe8 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -164,7 +164,9 @@ func runExtract(_ *cli.Context) error { g = NewSurgeon(ctx, api, pst) ) - driver := conformance.NewDriver(ctx, schema.Selector{}) + driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{ + DisableVMFlush: true, + }) // this is the root of the state tree we start with. root := incTs.ParentState() diff --git a/conformance/driver.go b/conformance/driver.go index ee9727cae..fdd4fe3a9 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -33,10 +33,24 @@ var ( type Driver struct { ctx context.Context selector schema.Selector + vmFlush bool } -func NewDriver(ctx context.Context, selector schema.Selector) *Driver { - return &Driver{ctx: ctx, selector: selector} +type DriverOpts struct { + // DisableVMFlush, when true, avoids calling VM.Flush(), forces a blockstore + // recursive copy, from the temporary buffer blockstore, to the real + // system's blockstore. Disabling VM flushing is useful when extracting test + // vectors and trimming state, as we don't want to force an accidental + // deep copy of the state tree. + // + // Disabling VM flushing almost always should go hand-in-hand with + // LOTUS_DISABLE_VM_BUF=iknowitsabadidea. That way, state tree writes are + // immediately committed to the blockstore. + DisableVMFlush bool +} + +func NewDriver(ctx context.Context, selector schema.Selector, opts DriverOpts) *Driver { + return &Driver{ctx: ctx, selector: selector, vmFlush: !opts.DisableVMFlush} } type ExecuteTipsetResult struct { @@ -155,12 +169,17 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch return nil, cid.Undef, err } - // do not flush the VM, as this forces a recursive copy to the blockstore, - // walking the full state tree, which we don't require. - // root, err := lvm.Flush(d.ctx) - // - // instead, flush the pending writes on the state tree. - root, err := lvm.StateTree().(*state.StateTree).Flush(d.ctx) + var root cid.Cid + if d.vmFlush { + // flush the VM, committing the state tree changes and forcing a + // recursive copoy from the temporary blcokstore to the real blockstore. + root, err = lvm.Flush(d.ctx) + } else { + // do not flush the VM, just the state tree; this should be used with + // LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible. + root, err = lvm.StateTree().(*state.StateTree).Flush(d.ctx) + } + return ret, root, err } diff --git a/conformance/runner.go b/conformance/runner.go index 1fc1c1425..6ffdcd2eb 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -43,7 +43,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { } // Create a new Driver. - driver := NewDriver(ctx, vector.Selector) + driver := NewDriver(ctx, vector.Selector, DriverOpts{}) // Apply every message. for i, m := range vector.ApplyMessages { @@ -93,7 +93,7 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { } // Create a new Driver. - driver := NewDriver(ctx, vector.Selector) + driver := NewDriver(ctx, vector.Selector, DriverOpts{}) // Apply every tipset. var receiptsIdx int From fe869c9c223e148b44efddca794a570145c1c3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 21:06:07 +0100 Subject: [PATCH 30/80] address review comments; lint. --- cmd/tvx/extract.go | 2 +- cmd/tvx/main.go | 4 +++- cmd/tvx/state.go | 2 +- cmd/tvx/stores.go | 4 +++- conformance/driver.go | 2 +- conformance/reporter.go | 8 ++++---- conformance/runner.go | 2 +- 7 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 06b3d6fe8..d3faa6d6d 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -304,7 +304,7 @@ func runExtract(_ *cli.Context) error { if err != nil { return err } - defer output.Close() + defer output.Close() //nolint:errcheck defer log.Printf("wrote test vector to file: %s", file) } diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 183c2fbe1..2ab217953 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -23,7 +23,9 @@ var apiEndpoint string var apiFlag = cli.StringFlag{ Name: "api", - Usage: "json-rpc api endpoint, formatted as token:multiaddr", + Usage: "json-rpc api endpoint, formatted as [token]:multiaddr;" + + "tvx uses unpriviliged operations, so the token may be omitted," + + "but permissions may change in the future", EnvVars: []string{"FULLNODE_API_INFO"}, DefaultText: "", Destination: &apiEndpoint, diff --git a/cmd/tvx/state.go b/cmd/tvx/state.go index cef5c5494..5bf1cf153 100644 --- a/cmd/tvx/state.go +++ b/cmd/tvx/state.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" "github.com/ipld/go-car" cbg "github.com/whyrusleeping/cbor-gen" diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go index 7d3fd5e3a..c389f8c88 100644 --- a/cmd/tvx/stores.go +++ b/cmd/tvx/stores.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/fatih/color" + dssync "github.com/ipfs/go-datastore/sync" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/blockstore" @@ -40,7 +41,8 @@ type Stores struct { // proxies Get requests for unknown CIDs to a Filecoin node, via the // ChainReadObj RPC. func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores { - ds := ds.NewMapDatastore() + ds := dssync.MutexWrap(ds.NewMapDatastore()) + ds = dssync.MutexWrap(ds) bs := &proxyingBlockstore{ ctx: ctx, diff --git a/conformance/driver.go b/conformance/driver.go index fdd4fe3a9..90d05ae88 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -33,7 +33,7 @@ var ( type Driver struct { ctx context.Context selector schema.Selector - vmFlush bool + vmFlush bool } type DriverOpts struct { diff --git a/conformance/reporter.go b/conformance/reporter.go index 747caae32..1cd2d389d 100644 --- a/conformance/reporter.go +++ b/conformance/reporter.go @@ -33,17 +33,17 @@ type LogReporter struct { var _ Reporter = (*LogReporter)(nil) -func (_ *LogReporter) Helper() {} +func (*LogReporter) Helper() {} -func (_ *LogReporter) Log(args ...interface{}) { +func (*LogReporter) Log(args ...interface{}) { log.Println(args...) } -func (_ *LogReporter) Logf(format string, args ...interface{}) { +func (*LogReporter) Logf(format string, args ...interface{}) { log.Printf(format, args...) } -func (_ *LogReporter) FailNow() { +func (*LogReporter) FailNow() { os.Exit(1) } diff --git a/conformance/runner.go b/conformance/runner.go index 6ffdcd2eb..456955b25 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -164,7 +164,7 @@ func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore. if err != nil { r.Fatalf("failed to write temporary state CAR: %s", err) } - defer os.RemoveAll(tmpCar) + defer os.RemoveAll(tmpCar) //nolint:errcheck color.NoColor = false // enable colouring. From 9a355c4bc5b10bd24c55f819e2a32ae882312dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 21:11:32 +0100 Subject: [PATCH 31/80] fix lint errors. --- cmd/tvx/main.go | 4 ++-- cmd/tvx/state.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 2ab217953..6d5c64101 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -22,8 +22,8 @@ import ( var apiEndpoint string var apiFlag = cli.StringFlag{ - Name: "api", - Usage: "json-rpc api endpoint, formatted as [token]:multiaddr;" + + Name: "api", + Usage: "json-rpc api endpoint, formatted as [token]:multiaddr;" + "tvx uses unpriviliged operations, so the token may be omitted," + "but permissions may change in the future", EnvVars: []string{"FULLNODE_API_INFO"}, diff --git a/cmd/tvx/state.go b/cmd/tvx/state.go index 5bf1cf153..bff5cbd6e 100644 --- a/cmd/tvx/state.go +++ b/cmd/tvx/state.go @@ -116,8 +116,8 @@ func (sg *StateSurgeon) GetAccessedActors(ctx context.Context, a api.FullNode, m recur = func(trace *types.ExecutionTrace) { accessed[trace.Msg.To] = struct{}{} accessed[trace.Msg.From] = struct{}{} - for _, s := range trace.Subcalls { - recur(&s) + for i := range trace.Subcalls { + recur(&trace.Subcalls[i]) } } recur(&trace.ExecutionTrace) From f05a40feed0ebd475ad5845ad751c449a01fb199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 12:27:42 +0100 Subject: [PATCH 32/80] use lotus CLI package; document API endpoint setting in usage. --- cmd/tvx/extract.go | 6 ++-- cmd/tvx/main.go | 80 ++++++++++++---------------------------------- 2 files changed, 23 insertions(+), 63 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index d3faa6d6d..d8053e5c2 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/conformance" "github.com/filecoin-project/go-address" @@ -38,7 +39,6 @@ var extractCmd = &cli.Command{ Description: "generate a message-class test vector by extracting it from a live chain", Action: runExtract, Flags: []cli.Flag{ - &apiFlag, &cli.StringFlag{ Name: "class", Usage: "class of vector to extract; other required flags depend on the; values: 'message'", @@ -72,7 +72,7 @@ var extractCmd = &cli.Command{ }, } -func runExtract(_ *cli.Context) error { +func runExtract(c *cli.Context) error { // LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering", // which stashes write operations in a BufferedBlockstore // (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21) @@ -91,7 +91,7 @@ func runExtract(_ *cli.Context) error { } // Make the API client. - api, closer, err := makeAPIClient() + api, closer, err := lcli.GetFullNodeAPI(c) if err != nil { return err } diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 6d5c64101..a74a029ce 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -1,49 +1,41 @@ package main import ( - "context" - "fmt" "log" - "net/http" "os" "sort" - "strings" - - "github.com/filecoin-project/go-jsonrpc" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" "github.com/urfave/cli/v2" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/api/client" ) -var apiEndpoint string - -var apiFlag = cli.StringFlag{ - Name: "api", - Usage: "json-rpc api endpoint, formatted as [token]:multiaddr;" + - "tvx uses unpriviliged operations, so the token may be omitted," + - "but permissions may change in the future", - EnvVars: []string{"FULLNODE_API_INFO"}, - DefaultText: "", - Destination: &apiEndpoint, -} - func main() { app := &cli.App{ Name: "tvx", Description: `tvx is a tool for extracting and executing test vectors. It has two subcommands. tvx extract extracts a test vector from a live network. It requires access to - a Filecoin client that exposes the standard JSON-RPC API endpoint. Set the API - endpoint on the FULLNODE_API_INFO env variable, or through the --api flag. The - format is token:multiaddr. Only message class test vectors are supported - for now. + a Filecoin client that exposes the standard JSON-RPC API endpoint. Only + message class test vectors are supported at this time. tvx exec executes test vectors against Lotus. Either you can supply one in a - file, or many as an ndjson stdin stream.`, + file, or many as an ndjson stdin stream. + + SETTING THE JSON-RPC API ENDPOINT + + You can set the JSON-RPC API endpoint through one of the following approaches. + + 1. Directly set the API endpoint on the FULLNODE_API_INFO env variable. + The format is [token]:multiaddr, where token is optional for commands not + accessing privileged operations. + + 2. If you're running tvx against a local Lotus client, you can set the REPO + env variable to have the API endpoint and token extracted from the repo. + + 3. Rely on the default fallback, which inspects ~/.lotus and extracts the + API endpoint string if the location is a Lotus repo. + + tvx will apply these approaches in the same order of precedence they're listed. +`, Usage: "tvx is a tool for extracting and executing test vectors", Commands: []*cli.Command{ extractCmd, @@ -60,35 +52,3 @@ func main() { log.Fatal(err) } } - -func makeAPIClient() (api.FullNode, jsonrpc.ClientCloser, error) { - sp := strings.SplitN(apiEndpoint, ":", 2) - if len(sp) != 2 { - return nil, nil, fmt.Errorf("invalid api value, missing token or address: %s", apiEndpoint) - } - - token := sp[0] - ma, err := multiaddr.NewMultiaddr(sp[1]) - if err != nil { - return nil, nil, fmt.Errorf("could not parse provided multiaddr: %w", err) - } - - _, dialAddr, err := manet.DialArgs(ma) - if err != nil { - return nil, nil, fmt.Errorf("invalid api multiAddr: %w", err) - } - - var ( - addr = "ws://" + dialAddr + "/rpc/v0" - headers = make(http.Header, 1) - ) - if len(token) != 0 { - headers.Add("Authorization", "Bearer "+token) - } - - node, closer, err := client.NewFullNodeRPC(context.Background(), addr, headers) - if err != nil { - return nil, nil, fmt.Errorf("could not connect to api: %w", err) - } - return node, closer, nil -} From dfdcbd184d7679cb32dfefe7be4ce2ffd6159d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 12:35:01 +0100 Subject: [PATCH 33/80] add --repo flag. --- cmd/tvx/extract.go | 3 ++- cmd/tvx/main.go | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index d8053e5c2..3408cea12 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -36,9 +36,10 @@ var extractFlags struct { var extractCmd = &cli.Command{ Name: "extract", - Description: "generate a message-class test vector by extracting it from a live chain", + Description: "generate a test vector by extracting it from a live chain", Action: runExtract, Flags: []cli.Flag{ + &repoFlag, &cli.StringFlag{ Name: "class", Usage: "class of vector to extract; other required flags depend on the; values: 'message'", diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index a74a029ce..361ba41c3 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -8,6 +8,18 @@ import ( "github.com/urfave/cli/v2" ) +// DefaultLotusRepoPath is where the fallback path where to look for a Lotus +// client repo. It is expanded with mitchellh/go-homedir, so it'll work with all +// OSes despite the Unix twiddle notation. +const DefaultLotusRepoPath = "~/.lotus" + +var repoFlag = cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Value: DefaultLotusRepoPath, + TakesFile: true, +} + func main() { app := &cli.App{ Name: "tvx", @@ -22,7 +34,7 @@ func main() { SETTING THE JSON-RPC API ENDPOINT - You can set the JSON-RPC API endpoint through one of the following approaches. + You can set the JSON-RPC API endpoint through one of the following methods. 1. Directly set the API endpoint on the FULLNODE_API_INFO env variable. The format is [token]:multiaddr, where token is optional for commands not @@ -30,11 +42,12 @@ func main() { 2. If you're running tvx against a local Lotus client, you can set the REPO env variable to have the API endpoint and token extracted from the repo. + Alternatively, you can pass the --repo CLI flag. 3. Rely on the default fallback, which inspects ~/.lotus and extracts the API endpoint string if the location is a Lotus repo. - tvx will apply these approaches in the same order of precedence they're listed. + tvx will apply these methods in the same order of precedence they're listed. `, Usage: "tvx is a tool for extracting and executing test vectors", Commands: []*cli.Command{ From 8f3be7866765ce5054dd2ffe20c55dee82a4fe2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 13:00:07 +0100 Subject: [PATCH 34/80] tvx/extract: allow passing in block to speed things up. --- cmd/tvx/extract.go | 71 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 3408cea12..09465d356 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -28,6 +28,7 @@ import ( var extractFlags struct { id string + block string class string cid string file string @@ -52,6 +53,11 @@ var extractCmd = &cli.Command{ Value: "(undefined)", Destination: &extractFlags.id, }, + &cli.StringFlag{ + Name: "block", + Usage: "optionally, the block CID the message was included in, to avoid expensive chain scanning", + Destination: &extractFlags.block, + }, &cli.StringFlag{ Name: "cid", Usage: "message CID to generate test vector from", @@ -98,27 +104,59 @@ func runExtract(c *cli.Context) error { } defer closer() - log.Printf("locating message with CID: %s", mcid) - - // Locate the message. - msgInfo, err := api.StateSearchMsg(ctx, mcid) - if err != nil { - return fmt.Errorf("failed to locate message: %w", err) - } - - log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) + var ( + msg *types.Message + incTs *types.TipSet + execTs *types.TipSet + ) // Extract the full message. - msg, err := api.ChainGetMessage(ctx, mcid) + msg, err = api.ChainGetMessage(ctx, mcid) if err != nil { return err } - log.Printf("full message: %+v", msg) + log.Printf("found message with CID %s: %+v", mcid, msg) - execTs, incTs, err := fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) - if err != nil { - return err + if block := extractFlags.block; block == "" { + log.Printf("locating message in blockchain") + + // Locate the message. + msgInfo, err := api.StateSearchMsg(ctx, mcid) + if err != nil { + return fmt.Errorf("failed to locate message: %w", err) + } + + log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) + + execTs, incTs, err = fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) + if err != nil { + return err + } + } else { + bcid, err := cid.Decode(block) + if err != nil { + return err + } + + log.Printf("message inclusion block CID was provided; scanning around it: %s", bcid) + + blk, err := api.ChainGetBlock(ctx, bcid) + if err != nil { + return fmt.Errorf("failed to get block: %w", err) + } + + // types.EmptyTSK hints to use the HEAD. + execTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height+1, types.EmptyTSK) + if err != nil { + return fmt.Errorf("failed to get message execution tipset: %w", err) + } + + // walk back from the execTs instead of HEAD, to save time. + incTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height, execTs.Key()) + if err != nil { + return fmt.Errorf("failed to get message inclusion tipset: %w", err) + } } log.Printf("message was executed in tipset: %s", execTs.Key()) @@ -274,7 +312,10 @@ func runExtract(c *cli.Context) error { Meta: &schema.Metadata{ ID: extractFlags.id, Gen: []schema.GenerationData{ - {Source: fmt.Sprintf("message:%s:%s", ntwkName, msg.Cid().String())}, + {Source: fmt.Sprintf("network:%s", ntwkName)}, + {Source: fmt.Sprintf("msg:%s", msg.Cid().String())}, + {Source: fmt.Sprintf("inc_ts:%s", incTs.Key().String())}, + {Source: fmt.Sprintf("exec_ts:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, CAR: out.Bytes(), From a0dffb44d31595a51857516a58439fd3964f044e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 13:11:34 +0100 Subject: [PATCH 35/80] tvx/extract: perform sanity check on receipt. --- cmd/tvx/extract.go | 31 ++++++++++++++++++++++++++++--- conformance/runner.go | 8 ++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 09465d356..8dbaf45d5 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -10,6 +10,8 @@ import ( "log" "os" + "github.com/fatih/color" + "github.com/filecoin-project/lotus/api" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" @@ -277,6 +279,25 @@ func runExtract(c *cli.Context) error { return fmt.Errorf("unknown state retention option: %s", retention) } + log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot) + log.Printf("performing sanity check on receipt") + + receipt := &schema.Receipt{ + ExitCode: int64(applyret.ExitCode), + ReturnValue: applyret.Return, + GasUsed: applyret.GasUsed, + } + + reporter := new(conformance.LogReporter) + conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed") + if reporter.Failed() { + log.Printf(color.RedString("receipt sanity check failed; aborting")) + return fmt.Errorf("vector generation aborted") + } else { + log.Printf(color.GreenString("receipt sanity check succeeded")) + } + + log.Printf("generating vector") msgBytes, err := msg.Serialize() if err != nil { return err @@ -311,11 +332,15 @@ func runExtract(c *cli.Context) error { Class: schema.ClassMessage, Meta: &schema.Metadata{ ID: extractFlags.id, + // TODO need to replace schema.GenerationData with a more flexible + // data structure that makes no assumption about the traceability + // data that's being recorded; a flexible map[string]string + // would do. Gen: []schema.GenerationData{ {Source: fmt.Sprintf("network:%s", ntwkName)}, - {Source: fmt.Sprintf("msg:%s", msg.Cid().String())}, - {Source: fmt.Sprintf("inc_ts:%s", incTs.Key().String())}, - {Source: fmt.Sprintf("exec_ts:%s", execTs.Key().String())}, + {Source: fmt.Sprintf("message:%s", msg.Cid().String())}, + {Source: fmt.Sprintf("inclusion_tipset:%s", incTs.Key().String())}, + {Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, CAR: out.Bytes(), diff --git a/conformance/runner.go b/conformance/runner.go index 456955b25..0fc4b13fc 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -65,7 +65,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { } // Assert that the receipt matches what the test vector expects. - assertMsgResult(r, vector.Post.Receipts[i], ret, strconv.Itoa(i)) + AssertMsgResult(r, vector.Post.Receipts[i], ret, strconv.Itoa(i)) } // Once all messages are applied, assert that the final state root matches @@ -105,7 +105,7 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { } for j, v := range ret.AppliedResults { - assertMsgResult(r, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i)) + AssertMsgResult(r, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i)) receiptsIdx++ } @@ -127,10 +127,10 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { } } -// assertMsgResult compares a message result. It takes the expected receipt +// AssertMsgResult compares a message result. It takes the expected receipt // encoded in the vector, the actual receipt returned by Lotus, and a message // label to log in the assertion failure message to facilitate debugging. -func assertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet, label string) { +func AssertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet, label string) { r.Helper() if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual { From 4c9717187f8995dd5b9cf647ac003815dd645a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 13:19:45 +0100 Subject: [PATCH 36/80] tvx/extract: small refactor. --- cmd/tvx/extract.go | 108 +++++++++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 8dbaf45d5..29330daaa 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -106,59 +106,9 @@ func runExtract(c *cli.Context) error { } defer closer() - var ( - msg *types.Message - incTs *types.TipSet - execTs *types.TipSet - ) - - // Extract the full message. - msg, err = api.ChainGetMessage(ctx, mcid) + msg, execTs, incTs, err := resolveFromChain(ctx, api, mcid) if err != nil { - return err - } - - log.Printf("found message with CID %s: %+v", mcid, msg) - - if block := extractFlags.block; block == "" { - log.Printf("locating message in blockchain") - - // Locate the message. - msgInfo, err := api.StateSearchMsg(ctx, mcid) - if err != nil { - return fmt.Errorf("failed to locate message: %w", err) - } - - log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) - - execTs, incTs, err = fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) - if err != nil { - return err - } - } else { - bcid, err := cid.Decode(block) - if err != nil { - return err - } - - log.Printf("message inclusion block CID was provided; scanning around it: %s", bcid) - - blk, err := api.ChainGetBlock(ctx, bcid) - if err != nil { - return fmt.Errorf("failed to get block: %w", err) - } - - // types.EmptyTSK hints to use the HEAD. - execTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height+1, types.EmptyTSK) - if err != nil { - return fmt.Errorf("failed to get message execution tipset: %w", err) - } - - // walk back from the execTs instead of HEAD, to save time. - incTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height, execTs.Key()) - if err != nil { - return fmt.Errorf("failed to get message inclusion tipset: %w", err) - } + return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err) } log.Printf("message was executed in tipset: %s", execTs.Key()) @@ -384,6 +334,60 @@ func runExtract(c *cli.Context) error { return nil } +// resolveFromChain queries the chain for the provided message, using the block CID to +// speed up the query, if provided +func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) { + // Extract the full message. + msg, err = api.ChainGetMessage(ctx, mcid) + if err != nil { + return nil, nil, nil, err + } + + log.Printf("found message with CID %s: %+v", mcid, msg) + + block := extractFlags.block + if block == "" { + log.Printf("locating message in blockchain") + + // Locate the message. + msgInfo, err := api.StateSearchMsg(ctx, mcid) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to locate message: %w", err) + } + + log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) + + execTs, incTs, err = fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) + return msg, execTs, incTs, err + } + + bcid, err := cid.Decode(block) + if err != nil { + return nil, nil, nil, err + } + + log.Printf("message inclusion block CID was provided; scanning around it: %s", bcid) + + blk, err := api.ChainGetBlock(ctx, bcid) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get block: %w", err) + } + + // types.EmptyTSK hints to use the HEAD. + execTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height+1, types.EmptyTSK) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get message execution tipset: %w", err) + } + + // walk back from the execTs instead of HEAD, to save time. + incTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height, execTs.Key()) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get message inclusion tipset: %w", err) + } + + return msg, execTs, incTs, nil +} + // fetchThisAndPrevTipset returns the full tipset identified by the key, as well // as the previous tipset. In the context of vector generation, the target // tipset is the one where a message was executed, and the previous tipset is From 9f6862a456e391552b6c17cd1a0c82d5a6e70965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 13:22:56 +0100 Subject: [PATCH 37/80] fix lint. --- cmd/tvx/extract.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 29330daaa..07c6ca7ef 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -230,7 +230,7 @@ func runExtract(c *cli.Context) error { } log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot) - log.Printf("performing sanity check on receipt") + log.Println("performing sanity check on receipt") receipt := &schema.Receipt{ ExitCode: int64(applyret.ExitCode), @@ -241,13 +241,13 @@ func runExtract(c *cli.Context) error { reporter := new(conformance.LogReporter) conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed") if reporter.Failed() { - log.Printf(color.RedString("receipt sanity check failed; aborting")) + log.Println(color.RedString("receipt sanity check failed; aborting")) return fmt.Errorf("vector generation aborted") - } else { - log.Printf(color.GreenString("receipt sanity check succeeded")) } - log.Printf("generating vector") + log.Println(color.GreenString("receipt sanity check succeeded")) + + log.Println("generating vector") msgBytes, err := msg.Serialize() if err != nil { return err From 96f882860fdc2a9edeb92f07964d7fb5a73b0c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 17:02:56 +0100 Subject: [PATCH 38/80] add extract-many command. --- cmd/tvx/actor_mapping.go | 44 +++++++++ cmd/tvx/extract.go | 50 ++++++---- cmd/tvx/extract_many.go | 204 +++++++++++++++++++++++++++++++++++++++ cmd/tvx/main.go | 6 +- 4 files changed, 283 insertions(+), 21 deletions(-) create mode 100644 cmd/tvx/actor_mapping.go create mode 100644 cmd/tvx/extract_many.go diff --git a/cmd/tvx/actor_mapping.go b/cmd/tvx/actor_mapping.go new file mode 100644 index 000000000..8c306aca0 --- /dev/null +++ b/cmd/tvx/actor_mapping.go @@ -0,0 +1,44 @@ +package main + +import ( + "reflect" + + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" +) + +var ActorMethodTable = make(map[string][]string, 64) + +var Actors = map[cid.Cid]interface{}{ + builtin.InitActorCodeID: builtin.MethodsInit, + builtin.CronActorCodeID: builtin.MethodsCron, + builtin.AccountActorCodeID: builtin.MethodsAccount, + builtin.StoragePowerActorCodeID: builtin.MethodsPower, + builtin.StorageMinerActorCodeID: builtin.MethodsMiner, + builtin.StorageMarketActorCodeID: builtin.MethodsMarket, + builtin.PaymentChannelActorCodeID: builtin.MethodsPaych, + builtin.MultisigActorCodeID: builtin.MethodsMultisig, + builtin.RewardActorCodeID: builtin.MethodsReward, + builtin.VerifiedRegistryActorCodeID: builtin.MethodsVerifiedRegistry, +} + +func init() { + for code, methods := range Actors { + cmh, err := multihash.Decode(code.Hash()) // identity hash. + if err != nil { + panic(err) + } + + var ( + aname = string(cmh.Digest) + rt = reflect.TypeOf(methods) + nf = rt.NumField() + ) + + ActorMethodTable[aname] = append(ActorMethodTable[aname], "Send") + for i := 0; i < nf; i++ { + ActorMethodTable[aname] = append(ActorMethodTable[aname], rt.Field(i).Name) + } + } +} diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 07c6ca7ef..e10fbad09 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -9,6 +9,7 @@ import ( "io" "log" "os" + "path/filepath" "github.com/fatih/color" @@ -28,7 +29,7 @@ import ( "github.com/urfave/cli/v2" ) -var extractFlags struct { +type extractOpts struct { id string block string class string @@ -37,6 +38,8 @@ var extractFlags struct { retain string } +var extractFlags extractOpts + var extractCmd = &cli.Command{ Name: "extract", Description: "generate a test vector by extracting it from a live chain", @@ -94,19 +97,23 @@ func runExtract(c *cli.Context) error { ctx := context.Background() - mcid, err := cid.Decode(extractFlags.cid) - if err != nil { - return err - } - // Make the API client. - api, closer, err := lcli.GetFullNodeAPI(c) + fapi, closer, err := lcli.GetFullNodeAPI(c) if err != nil { return err } defer closer() - msg, execTs, incTs, err := resolveFromChain(ctx, api, mcid) + return doExtract(ctx, fapi, extractFlags) +} + +func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { + mcid, err := cid.Decode(opts.cid) + if err != nil { + return err + } + + msg, execTs, incTs, err := resolveFromChain(ctx, fapi, mcid, opts.block) if err != nil { return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err) } @@ -119,7 +126,7 @@ func runExtract(c *cli.Context) error { // precursors, if any. var allmsgs []*types.Message for _, b := range incTs.Blocks() { - messages, err := api.ChainGetBlockMessages(ctx, b.Cid()) + messages, err := fapi.ChainGetBlockMessages(ctx, b.Cid()) if err != nil { return err } @@ -139,7 +146,7 @@ func runExtract(c *cli.Context) error { break } - log.Printf("message not found in block %s; precursors found: %v; ignoring block", b.Cid(), related) + log.Printf("message not found in block %s; number of precursors found: %d; ignoring block", b.Cid(), len(related)) } if allmsgs == nil { @@ -151,8 +158,8 @@ func runExtract(c *cli.Context) error { var ( // create a read-through store that uses ChainGetObject to fetch unknown CIDs. - pst = NewProxyingStores(ctx, api) - g = NewSurgeon(ctx, api, pst) + pst = NewProxyingStores(ctx, fapi) + g = NewSurgeon(ctx, fapi, pst) ) driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{ @@ -178,7 +185,7 @@ func runExtract(c *cli.Context) error { postroot cid.Cid applyret *vm.ApplyRet carWriter func(w io.Writer) error - retention = extractFlags.retain + retention = opts.retain ) log.Printf("using state retention strategy: %s", retention) @@ -204,7 +211,7 @@ func runExtract(c *cli.Context) error { case "accessed-actors": log.Printf("calculating accessed actors") // get actors accessed by message. - retain, err := g.GetAccessedActors(ctx, api, mcid) + retain, err := g.GetAccessedActors(ctx, fapi, mcid) if err != nil { return fmt.Errorf("failed to calculate accessed actors: %w", err) } @@ -267,12 +274,12 @@ func runExtract(c *cli.Context) error { return err } - version, err := api.Version(ctx) + version, err := fapi.Version(ctx) if err != nil { return err } - ntwkName, err := api.StateNetworkName(ctx) + ntwkName, err := fapi.StateNetworkName(ctx) if err != nil { return err } @@ -281,7 +288,7 @@ func runExtract(c *cli.Context) error { vector := schema.TestVector{ Class: schema.ClassMessage, Meta: &schema.Metadata{ - ID: extractFlags.id, + ID: opts.id, // TODO need to replace schema.GenerationData with a more flexible // data structure that makes no assumption about the traceability // data that's being recorded; a flexible map[string]string @@ -316,7 +323,11 @@ func runExtract(c *cli.Context) error { } output := io.WriteCloser(os.Stdout) - if file := extractFlags.file; file != "" { + if file := opts.file; file != "" { + dir := filepath.Dir(file) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("unable to create directory %s: %w", dir, err) + } output, err = os.Create(file) if err != nil { return err @@ -336,7 +347,7 @@ func runExtract(c *cli.Context) error { // resolveFromChain queries the chain for the provided message, using the block CID to // speed up the query, if provided -func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) { +func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid, block string) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) { // Extract the full message. msg, err = api.ChainGetMessage(ctx, mcid) if err != nil { @@ -345,7 +356,6 @@ func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid) (msg log.Printf("found message with CID %s: %+v", mcid, msg) - block := extractFlags.block if block == "" { log.Printf("locating message in blockchain") diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go new file mode 100644 index 000000000..8fea8df4c --- /dev/null +++ b/cmd/tvx/extract_many.go @@ -0,0 +1,204 @@ +package main + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/fatih/color" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/hashicorp/go-multierror" + "github.com/urfave/cli/v2" + + lcli "github.com/filecoin-project/lotus/cli" +) + +var extractManyFlags struct { + in string + outdir string +} + +var extractManyCmd = &cli.Command{ + Name: "extract-many", + Description: `generate many test vectors by repeateadly calling tvx extract, using a csv file as input. + + The CSV file must have a format just like the following: + + message_cid,receiver_code,method_num,exit_code,height,block_cid,seq + bafy2bzacedvuvgpsnwq7i7kltfap6hnp7fdmzf6lr4w34zycjrthb3v7k6zi6,fil/1/account,0,0,67972,bafy2bzacebthpxzlk7zhlkz3jfzl4qw7mdoswcxlf3rkof3b4mbxfj3qzfk7w,1 + bafy2bzacedwicofymn4imgny2hhbmcm4o5bikwnv3qqgohyx73fbtopiqlro6,fil/1/account,0,0,67860,bafy2bzacebj7beoxyzll522o6o76mt7von4psn3tlvunokhv4zhpwmfpipgti,2 + ... + + The first row MUST be a header row. At the bare minimum, those seven fields + must appear, in the order specified. Extra fields are accepted, but always + after these compulsory six. +`, + Action: runExtractMany, + Flags: []cli.Flag{ + &repoFlag, + &cli.StringFlag{ + Name: "in", + Usage: "path to input file (csv)", + Destination: &extractManyFlags.in, + }, + &cli.StringFlag{ + Name: "out-dir", + Usage: "output directory", + Destination: &extractManyFlags.outdir, + }, + }, +} + +func runExtractMany(c *cli.Context) error { + // LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering", + // which stashes write operations in a BufferedBlockstore + // (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21) + // such that they're not written until the VM is actually flushed. + // + // For some reason, the standard behaviour was not working for me (raulk), + // and disabling it (such that the state transformations are written immediately + // to the blockstore) worked. + _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") + + ctx := context.Background() + + // Make the API client. + fapi, closer, err := lcli.GetFullNodeAPI(c) + if err != nil { + return err + } + defer closer() + + var ( + in = extractManyFlags.in + outdir = extractManyFlags.outdir + ) + + if in == "" { + return fmt.Errorf("input file not provided") + } + + if outdir == "" { + return fmt.Errorf("output dir not provided") + } + + // Open the CSV file for reading. + f, err := os.Open(in) + if err != nil { + return fmt.Errorf("could not open file %s: %w", in, err) + } + + // Ensure the output directory exists. + if err := os.MkdirAll(outdir, 0755); err != nil { + return fmt.Errorf("could not create output dir %s: %w", outdir, err) + } + + // Create a CSV reader and validate the header row. + reader := csv.NewReader(f) + if header, err := reader.Read(); err != nil { + return fmt.Errorf("failed to read header from csv: %w", err) + } else if l := len(header); l < 7 { + return fmt.Errorf("insufficient number of fields: %d", l) + } else if f := header[0]; f != "message_cid" { + return fmt.Errorf("csv sanity check failed: expected first field in header to be 'message_cid'; was: %s", f) + } else { + log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header)) + } + + var generated []string + merr := new(multierror.Error) + // Read each row and extract the requested message. + for { + row, err := reader.Read() + if err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("failed to read row: %w", err) + } + var ( + cid = row[0] + actorcode = row[1] + methodnumstr = row[2] + exitcodestr = row[3] + _ = row[4] + block = row[5] + seq = row[6] + + exit int + methodnum int + methodname string + ) + + // Parse the exit code. + if exit, err = strconv.Atoi(exitcodestr); err != nil { + return fmt.Errorf("invalid exitcode number: %d", exit) + } + // Parse the method number. + if methodnum, err = strconv.Atoi(methodnumstr); err != nil { + return fmt.Errorf("invalid method number: %s", methodnumstr) + } + + // Lookup the method in actor method table. + if m, ok := ActorMethodTable[actorcode]; !ok { + return fmt.Errorf("unrecognized actor: %s", actorcode) + } else if methodnum >= len(m) { + return fmt.Errorf("unrecognized method number for actor %s: %d", actorcode, methodnum) + } else { + methodname = m[methodnum] + } + + // exitcode string representations are of kind ErrType(0); strip out + // the number portion. + exitcodename := strings.Split(exitcode.ExitCode(exit).String(), "(")[0] + // replace the slashes in the actor code name with underscores. + actorcodename := strings.ReplaceAll(actorcode, "/", "_") + + // Compute the ID of the vector. + id := fmt.Sprintf("extracted-msg-%s-%s-%s-%s", actorcodename, methodname, exitcodename, seq) + // Vector filename, using a base of outdir. + file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json" + + log.Println(color.YellowString("processing message id: %s", id)) + + opts := extractOpts{ + id: id, + block: block, + class: "message", + cid: cid, + file: file, + retain: "accessed-cids", + } + + if err := doExtract(ctx, fapi, opts); err != nil { + merr = multierror.Append(err, fmt.Errorf("failed to extract vector for message %s: %w", cid, err)) + continue + } + + log.Println(color.MagentaString("generated file: %s", file)) + + generated = append(generated, file) + } + + if len(generated) == 0 { + log.Println("no files generated") + } else { + log.Println("files generated:") + for _, g := range generated { + log.Println(g) + } + } + + if merr.ErrorOrNil() != nil { + log.Println(color.YellowString("done processing with errors: %s")) + } else { + log.Println(color.GreenString("done processing with no errors")) + } + + return merr.ErrorOrNil() +} diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 361ba41c3..6c887d163 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -23,7 +23,7 @@ var repoFlag = cli.StringFlag{ func main() { app := &cli.App{ Name: "tvx", - Description: `tvx is a tool for extracting and executing test vectors. It has two subcommands. + Description: `tvx is a tool for extracting and executing test vectors. It has three subcommands. tvx extract extracts a test vector from a live network. It requires access to a Filecoin client that exposes the standard JSON-RPC API endpoint. Only @@ -32,6 +32,9 @@ func main() { tvx exec executes test vectors against Lotus. Either you can supply one in a file, or many as an ndjson stdin stream. + tvx extract-many performs a batch extraction of many messages, supplied in a + CSV file. Refer to the help of that subcommand for more info. + SETTING THE JSON-RPC API ENDPOINT You can set the JSON-RPC API endpoint through one of the following methods. @@ -53,6 +56,7 @@ func main() { Commands: []*cli.Command{ extractCmd, execCmd, + extractManyCmd, }, } From 64f24fd276d0f964b78ece61fd2be6cdbb674b1f Mon Sep 17 00:00:00 2001 From: jennijuju Date: Mon, 28 Sep 2020 15:34:06 -0400 Subject: [PATCH 39/80] Added an option to hide sector info for `lotus-miner info` --- cmd/lotus-storage-miner/info.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 3ccfd67da..213d62e6e 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -33,6 +33,12 @@ var infoCmd = &cli.Command{ Subcommands: []*cli.Command{ infoAllCmd, }, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "hide-sectors-info", + Usage: "hide sectors info", + }, + }, Action: infoCmdAct, } @@ -199,10 +205,12 @@ func infoCmdAct(cctx *cli.Context) error { fmt.Printf("Expected Seal Duration: %s\n\n", sealdur) - fmt.Println("Sectors:") - err = sectorsInfo(ctx, nodeApi) - if err != nil { - return err + if !cctx.Bool("hide-sectors-info") { + fmt.Println("Sectors:") + err = sectorsInfo(ctx, nodeApi) + if err != nil { + return err + } } // TODO: grab actr state / info From 4b3b35c9defec5d2bfab6b527e4eeebe25109bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 23:04:52 +0100 Subject: [PATCH 40/80] conformance: record and feed circulating supply. --- cmd/tvx/extract.go | 18 ++++++++++++++---- conformance/driver.go | 34 ++++++++++++++++++++++------------ conformance/runner.go | 8 +++++++- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index e10fbad09..0dc7f6aa0 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -118,8 +118,17 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err) } + // get the circulating supply before the message was executed. + circSupplyDetail, err := fapi.StateCirculatingSupply(ctx, incTs.Key()) + if err != nil { + return fmt.Errorf("failed while fetching circulating supply: %w", err) + } + + circSupply := circSupplyDetail.FilCirculating.Int64() + log.Printf("message was executed in tipset: %s", execTs.Key()) log.Printf("message was included in tipset: %s", incTs.Key()) + log.Printf("circulating supply at inclusion tipset: %d", circSupply) log.Printf("finding precursor messages") // Iterate through blocks, finding the one that contains the message and its @@ -174,7 +183,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { log.Printf("number of precursors to apply: %d", len(precursors)) for i, m := range precursors { log.Printf("applying precursor %d, cid: %s", i, m.Cid()) - _, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m) + _, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m, &circSupplyDetail.FilCirculating) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) } @@ -199,7 +208,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { tbs.StartTracing() preroot = root - applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg) + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating) if err != nil { return fmt.Errorf("failed to execute message: %w", err) } @@ -224,7 +233,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { if err != nil { return err } - applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg) + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating) if err != nil { return fmt.Errorf("failed to execute message: %w", err) } @@ -302,7 +311,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { }, CAR: out.Bytes(), Pre: &schema.Preconditions{ - Epoch: int64(execTs.Height()), + Epoch: int64(execTs.Height()), + CircSupply: &circSupply, StateTree: &schema.StateTree{ RootCID: preroot, }, diff --git a/conformance/driver.go b/conformance/driver.go index 90d05ae88..3f50b67a9 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -3,8 +3,6 @@ package conformance import ( "context" - "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" @@ -15,6 +13,7 @@ import ( "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/test-vectors/schema" @@ -24,6 +23,11 @@ import ( ds "github.com/ipfs/go-datastore" ) +// DefaultCirculatingSupply is the fallback circulating supply returned by +// the driver's CircSupplyCalculator function, used if the vector specifies +// no circulating supply. +var DefaultCirculatingSupply = types.TotalFilecoinInt + var ( // BaseFee to use in the VM. // TODO make parametrisable through vector. @@ -136,18 +140,24 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot } // ExecuteMessage executes a conformance test vector message in a temporary VM. -func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message) (*vm.ApplyRet, cid.Cid, error) { - // dummy state manager; only to reference the GetNetworkVersion method, which does not depend on state. +func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message, circSupply *abi.TokenAmount) (*vm.ApplyRet, cid.Cid, error) { + // dummy state manager; only to reference the GetNetworkVersion method, + // which does not depend on state. sm := new(stmgr.StateManager) vmOpts := &vm.VMOpts{ - StateBase: preroot, - Epoch: epoch, - Rand: &testRand{}, // TODO always succeeds; need more flexibility. - Bstore: bs, - Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. - CircSupplyCalc: nil, - BaseFee: BaseFee, - NtwkVersion: sm.GetNtwkVersion, + StateBase: preroot, + Epoch: epoch, + Rand: &testRand{}, // TODO always succeeds; need more flexibility. + Bstore: bs, + Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. + CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) { + if circSupply != nil { + return *circSupply, nil + } + return DefaultCirculatingSupply, nil + }, + BaseFee: BaseFee, + NtwkVersion: sm.GetNtwkVersion, } lvm, err := vm.NewVM(context.TODO(), vmOpts) diff --git a/conformance/runner.go b/conformance/runner.go index 0fc4b13fc..812f3cc08 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -45,6 +45,12 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { // Create a new Driver. driver := NewDriver(ctx, vector.Selector, DriverOpts{}) + var circSupply *abi.TokenAmount + if cs := vector.Pre.CircSupply; cs != nil { + ta := abi.NewTokenAmount(*cs) + circSupply = &ta + } + // Apply every message. for i, m := range vector.ApplyMessages { msg, err := types.DecodeMessage(m.Bytes) @@ -59,7 +65,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { // Execute the message. var ret *vm.ApplyRet - ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg) + ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg, circSupply) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) } diff --git a/go.mod b/go.mod index b0de7dfd6..bf3748749 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.1 + github.com/filecoin-project/test-vectors/schema v0.0.2 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index bdb80aa83..6a310ed3d 100644 --- a/go.sum +++ b/go.sum @@ -258,8 +258,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= -github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.2 h1:/Pp//88WBXe0h+ksntdL2HpEgAmbwXrftAfeVG39zdY= +github.com/filecoin-project/test-vectors/schema v0.0.2/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From 9b403e26e5d6c279383469cc86e1ac554b50ec75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 23:07:45 +0100 Subject: [PATCH 41/80] fix lint. --- cmd/tvx/extract_many.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index 8fea8df4c..83ec72b21 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -26,7 +26,7 @@ var extractManyFlags struct { var extractManyCmd = &cli.Command{ Name: "extract-many", - Description: `generate many test vectors by repeateadly calling tvx extract, using a csv file as input. + Description: `generate many test vectors by repeatedly calling tvx extract, using a csv file as input. The CSV file must have a format just like the following: From 044674487e56c6355354191ab546a5410aee789e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 23:14:20 +0100 Subject: [PATCH 42/80] fix double mutex. --- cmd/tvx/stores.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go index c389f8c88..6e50e0839 100644 --- a/cmd/tvx/stores.go +++ b/cmd/tvx/stores.go @@ -42,14 +42,11 @@ type Stores struct { // ChainReadObj RPC. func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores { ds := dssync.MutexWrap(ds.NewMapDatastore()) - ds = dssync.MutexWrap(ds) - bs := &proxyingBlockstore{ ctx: ctx, api: api, Blockstore: blockstore.NewBlockstore(ds), } - return NewStores(ctx, ds, bs) } From eaece306b6c16efe642160f52d5e73a6a687d976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Sep 2020 11:17:23 +0200 Subject: [PATCH 43/80] wallet list cli: Print balances/nonces --- cli/wallet.go | 69 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/cli/wallet.go b/cli/wallet.go index 27993a1ba..0d69673f9 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -9,13 +9,16 @@ import ( "os" "strings" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/crypto" - types "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/urfave/cli/v2" "golang.org/x/xerrors" - "github.com/urfave/cli/v2" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/crypto" + + types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/lib/tablewriter" ) var walletCmd = &cli.Command{ @@ -66,6 +69,13 @@ var walletNew = &cli.Command{ var walletList = &cli.Command{ Name: "list", Usage: "List wallet address", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "addr-only", + Usage: "Only print addresses", + Aliases: []string{"a"}, + }, + }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -79,9 +89,54 @@ var walletList = &cli.Command{ return err } - for _, addr := range addrs { - fmt.Println(addr.String()) + def, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err } + + tw := tablewriter.New( + tablewriter.Col("Address"), + tablewriter.Col("Balance"), + tablewriter.Col("Nonce"), + tablewriter.Col("Default"), + tablewriter.NewLineCol("Error")) + + for _, addr := range addrs { + if cctx.Bool("addr-only") { + fmt.Println(addr.String()) + } else { + a, err := api.StateGetActor(ctx, addr, types.EmptyTSK) + if err != nil { + if !strings.Contains(err.Error(), "actor not found") { + tw.Write(map[string]interface{}{ + "Address": addr, + "Error": err, + }) + continue + } + + a = &types.Actor{ + Balance: big.Zero(), + } + } + + row := map[string]interface{}{ + "Address": addr, + "Balance": types.FIL(a.Balance), + "Nonce": a.Nonce, + } + if addr == def { + row["Default"] = "X" + } + + tw.Write(row) + } + } + + if !cctx.Bool("addr-only") { + return tw.Flush(os.Stdout) + } + return nil }, } From d1c10a61dd61a38a90c7d49dcd1cfeb74318eba4 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 29 Sep 2020 12:19:04 +0200 Subject: [PATCH 44/80] fix: message signer - always compare with mpool nonce --- chain/messagesigner/messagesigner.go | 44 +++++++++++++---------- chain/messagesigner/messagesigner_test.go | 5 ++- node/builder.go | 3 +- node/impl/full/mpool.go | 3 +- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index 41b0edee9..1ad83543b 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -4,21 +4,22 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/chain/wallet" - - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" + logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" ) const dsKeyActorNonce = "ActorNonce" +var log = logging.Logger("messagesigner") + type mpoolAPI interface { GetNonce(address.Address) (uint64, error) } @@ -67,30 +68,30 @@ func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message) (* // nextNonce increments the nonce. // If there is no nonce in the datastore, gets the nonce from the message pool. func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { - addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()}) + // Nonces used to be created by the mempool and we need to support nodes + // that have mempool nonces, so first check the mempool for a nonce for + // this address. Note that the mempool returns the actor state's nonce + // by default. + nonce, err := ms.mpool.GetNonce(addr) + if err != nil { + return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err) + } // Get the nonce for this address from the datastore - nonceBytes, err := ms.ds.Get(addrNonceKey) + addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()}) + dsNonceBytes, err := ms.ds.Get(addrNonceKey) - var nonce uint64 switch { case xerrors.Is(err, datastore.ErrNotFound): // If a nonce for this address hasn't yet been created in the - // datastore, check the mempool - nonces used to be created by - // the mempool so we need to support nodes that still have mempool - // nonces. Note that the mempool returns the actor state's nonce by - // default. - nonce, err = ms.mpool.GetNonce(addr) - if err != nil { - return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err) - } + // datastore, just use the nonce from the mempool case err != nil: return 0, xerrors.Errorf("failed to get nonce from datastore: %w", err) default: - // There is a nonce in the mempool, so unmarshall and increment it - maj, val, err := cbg.CborReadHeader(bytes.NewReader(nonceBytes)) + // There is a nonce in the datastore, so unmarshall and increment it + maj, val, err := cbg.CborReadHeader(bytes.NewReader(dsNonceBytes)) if err != nil { return 0, xerrors.Errorf("failed to parse nonce from datastore: %w", err) } @@ -98,7 +99,14 @@ func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { return 0, xerrors.Errorf("bad cbor type parsing nonce from datastore") } - nonce = val + 1 + dsNonce := val + 1 + + // The message pool nonce should be <= than the datastore nonce + if nonce <= dsNonce { + nonce = dsNonce + } else { + log.Warnf("mempool nonce was larger than datastore nonce (%d > %d)", nonce, dsNonce) + } } // Write the nonce for this address to the datastore diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index e52137892..55676b258 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -98,10 +98,9 @@ func TestMessageSignerSignMessage(t *testing.T) { To: to1, From: from1, }, - // Should ignore mpool nonce because after the first message nonce - // will come from the datastore + // Should adjust datastore nonce because mpool nonce is higher mpoolNonce: [1]uint64{10}, - expNonce: 6, + expNonce: 10, }}, }, { // Nonce should increment independently for each address diff --git a/node/builder.go b/node/builder.go index c49789a6a..da2924338 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,8 +6,6 @@ import ( "os" "time" - "github.com/filecoin-project/lotus/chain/messagesigner" - logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -37,6 +35,7 @@ import ( "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/messagepool" + "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 003260496..066aafdc5 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -4,14 +4,13 @@ import ( "context" "encoding/json" - "github.com/filecoin-project/lotus/chain/messagesigner" - "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) From 09e5cc90a40791b3139e5eac8912532246ca8058 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 29 Sep 2020 14:45:55 +0200 Subject: [PATCH 45/80] Add README to documentation/en with explanations --- documentation/en/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 documentation/en/README.md diff --git a/documentation/en/README.md b/documentation/en/README.md new file mode 100644 index 000000000..76f11ed90 --- /dev/null +++ b/documentation/en/README.md @@ -0,0 +1,16 @@ +# Lotus documentation + +This folder contains some Lotus documentation mostly intended for Lotus developers. + +User documentation (including documentation for miners) has been moved to specific Lotus sections in https://docs.filecoin.io: + +- https://docs.filecoin.io/get-started/lotus +- https://docs.filecoin.io/store/lotus +- https://docs.filecoin.io/mine/lotus +- https://docs.filecoin.io/build/lotus + +## The Lotu.sh site + +The https://lotu.sh and https://docs.lotu.sh sites are generated from this folder based on the index provided by [.library.json](.library.json). This is done at the [lotus-docs repository](https://github.com/filecoin-project/lotus-docs), which contains Lotus as a git submodule. + +To update the site, the lotus-docs repository should be updated with the desired version for the lotus git submodule. Once pushed to master, it will be auto-deployed. From 96193c20448a530e3f0c542700fb78fa928bc880 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 16 Sep 2020 16:34:54 +0200 Subject: [PATCH 46/80] Implement bench-cache Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/caching_verifier.go | 33 +++++++++++++++ cmd/lotus-bench/import.go | 63 +++++++++++++++++++++-------- 2 files changed, 80 insertions(+), 16 deletions(-) create mode 100644 cmd/lotus-bench/caching_verifier.go diff --git a/cmd/lotus-bench/caching_verifier.go b/cmd/lotus-bench/caching_verifier.go new file mode 100644 index 000000000..cd794e647 --- /dev/null +++ b/cmd/lotus-bench/caching_verifier.go @@ -0,0 +1,33 @@ +package main + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" + "github.com/ipfs/go-datastore" +) + +type cachingVerifier struct { + ds datastore.Datastore + backend ffiwrapper.Verifier +} + +func (cv *cachingVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { + svi.MarshalCBOR(nil) + return cv.backend.VerifySeal(svi) +} +func (cv *cachingVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) { + info.MarshalCBOR(nil) + return cv.backend.VerifyWinningPoSt(ctx, info) +} +func (cv *cachingVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) { + info.MarshalCBOR(nil) + return cv.backend.VerifyWindowPoSt(ctx, info) +} +func (cv *cachingVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, a abi.ActorID, rnd abi.PoStRandomness, u uint64) ([]uint64, error) { + return cv.backend.GenerateWinningPoStSectorChallenge(ctx, proofType, a, rnd, u) +} + +var _ ffiwrapper.Verifier = (*cachingVerifier)(nil) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index f2845ba20..fc81c600e 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -60,6 +60,15 @@ var importBenchCmd = &cli.Command{ Name: "repodir", Usage: "set the repo directory for the lotus bench run (defaults to /tmp)", }, + &cli.StringFlag{ + Name: "syscall-cache", + Usage: "read and write syscall results from datastore", + }, + &cli.BoolFlag{ + Name: "export-traces", + Usage: "should we export execution traces", + Value: true, + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -85,7 +94,10 @@ var importBenchCmd = &cli.Command{ tdir = tmp } - bds, err := badger.NewDatastore(tdir, nil) + bdgOpt := badger.DefaultOptions + bdgOpt.GcInterval = 0 + + bds, err := badger.NewDatastore(tdir, &bdgOpt) if err != nil { return err } @@ -96,7 +108,21 @@ var importBenchCmd = &cli.Command{ } bs = cbs ds := datastore.NewMapDatastore() - cs := store.NewChainStore(bs, ds, vm.Syscalls(ffiwrapper.ProofVerifier)) + + var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier + if cctx.IsSet("syscall-cache") { + + scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &bdgOpt) + if err != nil { + return xerrors.Errorf("opening syscall-cache datastore: %w", err) + } + verifier = &cachingVerifier{ + ds: scds, + backend: verifier, + } + } + + cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier)) stm := stmgr.NewStateManager(cs) prof, err := os.Create("import-bench.prof") @@ -144,13 +170,16 @@ var importBenchCmd = &cli.Command{ ts = next } - ibj, err := os.Create("import-bench.json") - if err != nil { - return err - } - defer ibj.Close() //nolint:errcheck + var enc *json.Encoder + if cctx.Bool("export-traces") { + ibj, err := os.Create("import-bench.json") + if err != nil { + return err + } + defer ibj.Close() //nolint:errcheck - enc := json.NewEncoder(ibj) + enc = json.NewEncoder(ibj) + } var lastTse *TipSetExec @@ -173,17 +202,19 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } - stripCallers(trace) + if enc != nil { + stripCallers(trace) - lastTse = &TipSetExec{ - TipSet: cur.Key(), - Trace: trace, - Duration: time.Since(start), + lastTse = &TipSetExec{ + TipSet: cur.Key(), + Trace: trace, + Duration: time.Since(start), + } + if err := enc.Encode(lastTse); err != nil { + return xerrors.Errorf("failed to write out tipsetexec: %w", err) + } } lastState = st - if err := enc.Encode(lastTse); err != nil { - return xerrors.Errorf("failed to write out tipsetexec: %w", err) - } } pprof.StopCPUProfile() From 79ba4598d6f85bdc3e95faeb0e394af61e5e0666 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 16 Sep 2020 17:54:22 +0200 Subject: [PATCH 47/80] Implement cache Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/caching_verifier.go | 75 +++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-bench/caching_verifier.go b/cmd/lotus-bench/caching_verifier.go index cd794e647..28897071a 100644 --- a/cmd/lotus-bench/caching_verifier.go +++ b/cmd/lotus-bench/caching_verifier.go @@ -1,12 +1,16 @@ package main import ( + "bufio" "context" + "errors" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/ipfs/go-datastore" + "github.com/minio/blake2b-simd" + cbg "github.com/whyrusleeping/cbor-gen" ) type cachingVerifier struct { @@ -14,17 +18,78 @@ type cachingVerifier struct { backend ffiwrapper.Verifier } +const bufsize = 128 + +func (cv cachingVerifier) withCache(execute func() (bool, error), param cbg.CBORMarshaler) (bool, error) { + hasher := blake2b.New256() + wr := bufio.NewWriterSize(hasher, bufsize) + err := param.MarshalCBOR(wr) + if err != nil { + log.Errorf("could not marshal call info: %+v", err) + return execute() + } + err = wr.Flush() + if err != nil { + log.Errorf("could not flush: %+v", err) + return execute() + } + hash := hasher.Sum(nil) + key := datastore.NewKey(string(hash)) + fromDs, err := cv.ds.Get(key) + if err == nil { + switch fromDs[0] { + case 's': + return true, nil + case 'f': + return false, nil + case 'e': + return false, errors.New(string(fromDs[1:])) + default: + log.Errorf("bad cached result in cache %s(%x)", fromDs[0], fromDs[0]) + return execute() + } + } else if errors.Is(err, datastore.ErrNotFound) { + // recalc + ok, err := execute() + var save []byte + if err != nil { + if ok { + log.Errorf("sucess with an error: %+v", err) + } else { + save = append([]byte{'e'}, []byte(err.Error())...) + } + } else if ok { + save = []byte{'s'} + } else { + save = []byte{'f'} + } + + if len(save) != 0 { + errSave := cv.ds.Put(key, save) + if errSave != nil { + log.Errorf("error saving result: %+v", errSave) + } + } + + return ok, err + } else { + log.Errorf("could not get data from cache: %+v", err) + return execute() + } +} + func (cv *cachingVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { - svi.MarshalCBOR(nil) - return cv.backend.VerifySeal(svi) + return cv.withCache(func() (bool, error) { + return cv.backend.VerifySeal(svi) + }, &svi) } func (cv *cachingVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) { - info.MarshalCBOR(nil) return cv.backend.VerifyWinningPoSt(ctx, info) } func (cv *cachingVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) { - info.MarshalCBOR(nil) - return cv.backend.VerifyWindowPoSt(ctx, info) + return cv.withCache(func() (bool, error) { + return cv.backend.VerifyWindowPoSt(ctx, info) + }, &info) } func (cv *cachingVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, a abi.ActorID, rnd abi.PoStRandomness, u uint64) ([]uint64, error) { return cv.backend.GenerateWinningPoStSectorChallenge(ctx, proofType, a, rnd, u) From 53ab17cf50f5d7a69b81a6945bbe317a747bd10d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 16 Sep 2020 20:10:00 +0200 Subject: [PATCH 48/80] Add no import to import-bench Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index fc81c600e..c16796a13 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/lotus/lib/blockstore" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" + "github.com/ipld/go-car" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -69,6 +70,10 @@ var importBenchCmd = &cli.Command{ Usage: "should we export execution traces", Value: true, }, + &cli.BoolFlag{ + Name: "no-import", + Usage: "should we import the chain? if set to true chain has to be previously imported", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -111,7 +116,6 @@ var importBenchCmd = &cli.Command{ var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier if cctx.IsSet("syscall-cache") { - scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &bdgOpt) if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) @@ -135,9 +139,21 @@ var importBenchCmd = &cli.Command{ return err } - head, err := cs.Import(cfi) - if err != nil { - return err + var head *types.TipSet + if !cctx.Bool("no-import") { + head, err = cs.Import(cfi) + if err != nil { + return err + } + } else { + cr, err := car.NewCarReader(cfi) + if err != nil { + return err + } + head, err = cs.LoadTipSet(types.NewTipSetKey(cr.Header.Roots...)) + if err != nil { + return err + } } gb, err := cs.GetTipsetByHeight(context.TODO(), 0, head, true) @@ -188,6 +204,7 @@ var importBenchCmd = &cli.Command{ cur := tschain[i] log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids()) if cur.ParentState() != lastState { + stripCallers(lastTse.Trace) lastTrace := lastTse.Trace d, err := json.MarshalIndent(lastTrace, "", " ") if err != nil { @@ -202,14 +219,14 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } + lastTse = &TipSetExec{ + TipSet: cur.Key(), + Trace: trace, + Duration: time.Since(start), + } if enc != nil { - stripCallers(trace) + stripCallers(lastTse.Trace) - lastTse = &TipSetExec{ - TipSet: cur.Key(), - Trace: trace, - Duration: time.Since(start), - } if err := enc.Encode(lastTse); err != nil { return xerrors.Errorf("failed to write out tipsetexec: %w", err) } From 1f4d1dcc58fc2da3574c479159775e4b19b1a720 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 16 Sep 2020 20:38:28 +0200 Subject: [PATCH 49/80] Do not sync Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index c16796a13..443e67950 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -101,6 +101,8 @@ var importBenchCmd = &cli.Command{ bdgOpt := badger.DefaultOptions bdgOpt.GcInterval = 0 + bdgOpt.Options.SyncWrites = false + bdgOpt.Options.Truncate = true bds, err := badger.NewDatastore(tdir, &bdgOpt) if err != nil { From 12a0dd3d0a0a6b80c1e11c2e05d39b5024d55139 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 17 Sep 2020 00:06:20 +0200 Subject: [PATCH 50/80] <3 to linter Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/caching_verifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-bench/caching_verifier.go b/cmd/lotus-bench/caching_verifier.go index 28897071a..51ab696f7 100644 --- a/cmd/lotus-bench/caching_verifier.go +++ b/cmd/lotus-bench/caching_verifier.go @@ -54,7 +54,7 @@ func (cv cachingVerifier) withCache(execute func() (bool, error), param cbg.CBOR var save []byte if err != nil { if ok { - log.Errorf("sucess with an error: %+v", err) + log.Errorf("success with an error: %+v", err) } else { save = append([]byte{'e'}, []byte(err.Error())...) } From 108fe7823c9dadf3d7e3812d79a66f4e6769ef5e Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Sep 2020 13:39:38 +0200 Subject: [PATCH 51/80] Add command to trigger gc Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 443e67950..72ac5d60b 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -74,6 +74,9 @@ var importBenchCmd = &cli.Command{ Name: "no-import", Usage: "should we import the chain? if set to true chain has to be previously imported", }, + &cli.BoolFlag{ + Name: "only-gc", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -103,11 +106,15 @@ var importBenchCmd = &cli.Command{ bdgOpt.GcInterval = 0 bdgOpt.Options.SyncWrites = false bdgOpt.Options.Truncate = true + bdgOpt.Options.DetectConflicts = false + bdgOpt.Options.MaxTableSize = 64 << 20 bds, err := badger.NewDatastore(tdir, &bdgOpt) if err != nil { return err } + + bds.CollectGarbage() bs := blockstore.NewBlockstore(bds) cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts()) if err != nil { @@ -122,11 +129,15 @@ var importBenchCmd = &cli.Command{ if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) } + scds.CollectGarbage() verifier = &cachingVerifier{ ds: scds, backend: verifier, } } + if cctx.Bool("only-gc") { + return nil + } cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier)) stm := stmgr.NewStateManager(cs) From 782717948ae9926e4afa7a42f758e8c70abe13d8 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Sep 2020 13:54:20 +0200 Subject: [PATCH 52/80] Add logs Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 72ac5d60b..c6d49a78c 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -114,7 +114,11 @@ var importBenchCmd = &cli.Command{ return err } - bds.CollectGarbage() + if cctx.Bool("only-gc") { + log.Info("calling CollectGarbage on main ds") + bds.CollectGarbage() + log.Info("done calling CollectGarbage on main ds") + } bs := blockstore.NewBlockstore(bds) cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts()) if err != nil { @@ -129,7 +133,12 @@ var importBenchCmd = &cli.Command{ if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) } - scds.CollectGarbage() + + if cctx.Bool("only-gc") { + log.Info("calling CollectGarbage on syscall ds") + scds.CollectGarbage() + log.Info("done calling CollectGarbage on syscall ds") + } verifier = &cachingVerifier{ ds: scds, backend: verifier, From 3858309368bfa069f86f71b3558ef4532d6f422f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Sep 2020 13:55:37 +0200 Subject: [PATCH 53/80] Add http to import bench Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index c6d49a78c..8c874206f 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -91,6 +91,10 @@ var importBenchCmd = &cli.Command{ } defer cfi.Close() //nolint:errcheck // read only file + go func() { + http.ListenAndServe("localhost:6060", nil) //nolint:errcheck + }() + var tdir string if rdir := cctx.String("repodir"); rdir != "" { tdir = rdir From 01386a206c0c8c9d9007d35a80307cae93b0781b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Sep 2020 15:17:13 +0200 Subject: [PATCH 54/80] Update options Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 8c874206f..db87ebeb6 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -29,8 +29,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + bdg "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger2" + "github.com/urfave/cli/v2" "golang.org/x/xerrors" ) @@ -108,10 +110,10 @@ var importBenchCmd = &cli.Command{ bdgOpt := badger.DefaultOptions bdgOpt.GcInterval = 0 + bdgOpt.Options = bdg.DefaultOptions("") bdgOpt.Options.SyncWrites = false bdgOpt.Options.Truncate = true bdgOpt.Options.DetectConflicts = false - bdgOpt.Options.MaxTableSize = 64 << 20 bds, err := badger.NewDatastore(tdir, &bdgOpt) if err != nil { From f21c5cbbe28fe382c1670298e11819cf4949e7af Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 19:11:37 +0200 Subject: [PATCH 55/80] Add start-at Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 69 ++++++++++++++++++++++++--------------- go.mod | 9 ++--- go.sum | 31 ++++++++++++++++++ 3 files changed, 78 insertions(+), 31 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index db87ebeb6..3c5bd0cda 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/statediff" bdg "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" @@ -79,6 +80,9 @@ var importBenchCmd = &cli.Command{ &cli.BoolFlag{ Name: "only-gc", }, + &cli.Int64Flag{ + Name: "start-at", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -194,6 +198,20 @@ var importBenchCmd = &cli.Command{ return err } + startEpoch := abi.ChainEpoch(1) + if cctx.IsSet("start-at") { + startEpoch = abi.ChainEpoch(cctx.Int64("start-at")) + start, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(cctx.Int64("start-at")), head, true) + if err != nil { + return err + } + + err = cs.SetHead(start) + if err != nil { + return err + } + } + if h := cctx.Int64("height"); h != 0 { tsh, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), head, true) if err != nil { @@ -204,7 +222,7 @@ var importBenchCmd = &cli.Command{ ts := head tschain := []*types.TipSet{ts} - for ts.Height() != 0 { + for ts.Height() > startEpoch { next, err := cs.LoadTipSet(ts.Parents()) if err != nil { return err @@ -225,41 +243,38 @@ var importBenchCmd = &cli.Command{ enc = json.NewEncoder(ibj) } - var lastTse *TipSetExec - - lastState := tschain[len(tschain)-1].ParentState() - for i := len(tschain) - 2; i >= 0; i-- { + for i := len(tschain) - 1; i >= 1; i-- { cur := tschain[i] + start := time.Now() log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids()) - if cur.ParentState() != lastState { - stripCallers(lastTse.Trace) - lastTrace := lastTse.Trace + st, trace, err := stm.ExecutionTrace(context.TODO(), cur) + if err != nil { + return err + } + tse := &TipSetExec{ + TipSet: cur.Key(), + Trace: trace, + Duration: time.Since(start), + } + if enc != nil { + stripCallers(tse.Trace) + + if err := enc.Encode(tse); err != nil { + return xerrors.Errorf("failed to write out tipsetexec: %w", err) + } + } + if tschain[i-1].ParentState() != st { + stripCallers(tse.Trace) + lastTrace := tse.Trace d, err := json.MarshalIndent(lastTrace, "", " ") if err != nil { panic(err) } fmt.Println("TRACE") fmt.Println(string(d)) - return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), lastState) + fmt.Println(statediff.Diff(context.Background(), bs, tschain[i-1].ParentState(), st, statediff.ExpandActors)) + return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), st) } - start := time.Now() - st, trace, err := stm.ExecutionTrace(context.TODO(), cur) - if err != nil { - return err - } - lastTse = &TipSetExec{ - TipSet: cur.Key(), - Trace: trace, - Duration: time.Since(start), - } - if enc != nil { - stripCallers(lastTse.Trace) - - if err := enc.Encode(lastTse); err != nil { - return xerrors.Errorf("failed to write out tipsetexec: %w", err) - } - } - lastState = st } pprof.StopCPUProfile() diff --git a/go.mod b/go.mod index 2c0322ecc..83137d7c8 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/GeertJohan/go.rice v1.0.0 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 github.com/coreos/go-systemd/v22 v22.0.0 @@ -38,10 +37,10 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 + github.com/filecoin-project/statediff v0.0.6-0.20200918150628-da86dd0d264c github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 - github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 @@ -117,7 +116,6 @@ require ( github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 go.opencensus.io v0.22.4 - go.uber.org/dig v1.10.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 @@ -127,9 +125,12 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 gopkg.in/cheggaaa/pb.v1 v1.0.28 gotest.tools v2.2.0+incompatible - launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) +replace github.com/filecoin-project/lotus => ./ + +replace github.com/filecoin-project/statediff => ./../statediff + replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi diff --git a/go.sum b/go.sum index 05e643708..2455766b1 100644 --- a/go.sum +++ b/go.sum @@ -159,6 +159,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= @@ -206,6 +208,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanw/esbuild v0.6.28/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -218,6 +221,8 @@ github.com/filecoin-project/go-address v0.0.4 h1:gSNMv0qWwH16fGQs7ycOUrDjY6YCSsg github.com/filecoin-project/go-address v0.0.4/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= +github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161 h1:K6t4Hrs+rwUxBz2xg88Bdqeh4k5/rycQFdPseZhRyfE= +github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -280,6 +285,7 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= @@ -506,6 +512,11 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.2.0 h1:x94MvHLNuRwBlZzVal7tR1RYK7T7H6bqQLPopxDbIF0= +github.com/ipfs/go-graphsync v0.2.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.1.2 h1:25Ll9kIXCE+DY0dicvfS3KMw+U5sd01b/FJbA7KAbhg= +github.com/ipfs/go-graphsync v0.1.2/go.mod h1:sLXVXm1OxtE2XYPw62MuXCdAuNwkAdsbnfrmos5odbA= +github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= @@ -688,6 +699,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= @@ -1015,6 +1027,8 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1317,6 +1331,8 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1369,6 +1385,7 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200723185710-6a3894a6352b/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee h1:U7zWWvvAjT76EiuWPSOiZlQDnaQYPxPoxugTtTAcJK0= @@ -1395,6 +1412,8 @@ github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d/go.mod h1:g7c github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= +github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345 h1:IJVAwIctqDFOrO0C2qzksXmANviyHJzrklU27e1ltzE= +github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345/go.mod h1:D7hA8H5pyQx7Y5Em7IWx1R4vNJzfon3gpG9nxjkITjQ= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829 h1:wb7xrDzfkLgPHsSEBm+VSx6aDdi64VtV0xvP0E6j8bk= @@ -1483,6 +1502,7 @@ golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1495,6 +1515,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1508,11 +1530,14 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -1614,6 +1639,7 @@ golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1623,6 +1649,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1631,6 +1658,7 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= @@ -1674,6 +1702,7 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1802,6 +1831,8 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= From ebc8489ff183859cc0d567bd95b5904183b25152 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 20:27:24 +0200 Subject: [PATCH 56/80] Add global-profile option Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 3c5bd0cda..e201baa85 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -80,6 +80,10 @@ var importBenchCmd = &cli.Command{ &cli.BoolFlag{ Name: "only-gc", }, + &cli.BoolFlag{ + Name: "global-profile", + Value: true, + }, &cli.Int64Flag{ Name: "start-at", }, @@ -161,14 +165,16 @@ var importBenchCmd = &cli.Command{ cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier)) stm := stmgr.NewStateManager(cs) - prof, err := os.Create("import-bench.prof") - if err != nil { - return err - } - defer prof.Close() //nolint:errcheck + if cctx.Bool("global-profile") { + prof, err := os.Create("import-bench.prof") + if err != nil { + return err + } + defer prof.Close() //nolint:errcheck - if err := pprof.StartCPUProfile(prof); err != nil { - return err + if err := pprof.StartCPUProfile(prof); err != nil { + return err + } } var head *types.TipSet From 35cf69ae646574557acfaaa8bd65a4693560b352 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 20:49:40 +0200 Subject: [PATCH 57/80] Disable bloomcache Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 3 +++ go.mod | 1 + 2 files changed, 4 insertions(+) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index e201baa85..5da4f2d3d 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -134,6 +134,9 @@ var importBenchCmd = &cli.Command{ log.Info("done calling CollectGarbage on main ds") } bs := blockstore.NewBlockstore(bds) + cacheOpts := blockstore.DefaultCacheOpts() + cacheOpts.HasBloomFilterSize = 0 + cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts()) if err != nil { return err diff --git a/go.mod b/go.mod index 83137d7c8..d2e552ffc 100644 --- a/go.mod +++ b/go.mod @@ -116,6 +116,7 @@ require ( github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 go.opencensus.io v0.22.4 + go.uber.org/dig v1.10.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 From 242a77b391c001ddf3fad53c3f5c8160e157c4c3 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 20:50:06 +0200 Subject: [PATCH 58/80] go mod tidy Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 2 +- go.mod | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 5da4f2d3d..428fd4212 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -137,7 +137,7 @@ var importBenchCmd = &cli.Command{ cacheOpts := blockstore.DefaultCacheOpts() cacheOpts.HasBloomFilterSize = 0 - cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts()) + cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, cacheOpts) if err != nil { return err } diff --git a/go.mod b/go.mod index d2e552ffc..83137d7c8 100644 --- a/go.mod +++ b/go.mod @@ -116,7 +116,6 @@ require ( github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 go.opencensus.io v0.22.4 - go.uber.org/dig v1.10.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 From b7f18b460147f43d59203f43f5845164d9f81066 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 21:00:38 +0200 Subject: [PATCH 59/80] Disable callers Signed-off-by: Jakub Sztandera --- chain/vm/runtime.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 156d57282..eb4476718 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -5,7 +5,6 @@ import ( "context" "encoding/binary" "fmt" - gruntime "runtime" "time" "github.com/filecoin-project/go-address" @@ -493,7 +492,8 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) { func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError { toUse := gas.Total() var callers [10]uintptr - cout := gruntime.Callers(2+skip, callers[:]) + + cout := 0 //gruntime.Callers(2+skip, callers[:]) now := build.Clock.Now() if rt.lastGasCharge != nil { From 1c6214b76d1cb1022afeb8ac66e475c046ec4a4f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 20 Sep 2020 03:22:41 +0200 Subject: [PATCH 60/80] Usage go-bitfield with buffer pool Signed-off-by: Jakub Sztandera --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 83137d7c8..afa317614 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/fatih/color v1.8.0 github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d github.com/filecoin-project/go-address v0.0.4 - github.com/filecoin-project/go-bitfield v0.2.0 + github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.6.6 diff --git a/go.sum b/go.sum index 2455766b1..28606d17a 100644 --- a/go.sum +++ b/go.sum @@ -225,6 +225,10 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161 github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= +github.com/filecoin-project/go-bitfield v0.2.1-0.20200920171219-7c2059195a8c h1:eEmdVMWo7AngX9fGZSSAm/V6+7tqiBawFfHRjW35JwU= +github.com/filecoin-project/go-bitfield v0.2.1-0.20200920171219-7c2059195a8c/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= +github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g= +github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= From 0771c23fb02d2c6cb456f81c297cdb441345e7a6 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 21 Sep 2020 22:47:03 +0200 Subject: [PATCH 61/80] Use pebble Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 26 ++++++++++++++++++++++++-- go.mod | 4 +++- go.sum | 23 +++++++++++++++++++++-- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 428fd4212..b535ed96c 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -16,6 +16,8 @@ import ( "sort" "time" + "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/bloom" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" @@ -33,6 +35,7 @@ import ( bdg "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger2" + pebbleds "github.com/ipfs/go-ds-pebble" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -123,14 +126,33 @@ var importBenchCmd = &cli.Command{ bdgOpt.Options.Truncate = true bdgOpt.Options.DetectConflicts = false - bds, err := badger.NewDatastore(tdir, &bdgOpt) + cache := 512 + bds, err := pebbleds.NewDatastore(tdir, &pebble.Options{ + // Pebble has a single combined cache area and the write + // buffers are taken from this too. Assign all available + // memory allowance for cache. + Cache: pebble.NewCache(int64(cache * 1024 * 1024)), + // The size of memory table(as well as the write buffer). + // Note, there may have more than two memory tables in the system. + // MemTableStopWritesThreshold can be configured to avoid the memory abuse. + MemTableSize: cache * 1024 * 1024 / 4, + // The default compaction concurrency(1 thread), + // Here use all available CPUs for faster compaction. + MaxConcurrentCompactions: runtime.NumCPU(), + // Per-level options. Options for at least one level must be specified. The + // options for the last level are used for all subsequent levels. + Levels: []pebble.LevelOptions{ + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + }, + Logger: log, + }) if err != nil { return err } if cctx.Bool("only-gc") { log.Info("calling CollectGarbage on main ds") - bds.CollectGarbage() + //bds.CollectGarbage() log.Info("done calling CollectGarbage on main ds") } bs := blockstore.NewBlockstore(bds) diff --git a/go.mod b/go.mod index afa317614..3b3b4fed7 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 + github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b github.com/coreos/go-systemd/v22 v22.0.0 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e github.com/dgraph-io/badger/v2 v2.2007.2 @@ -53,10 +54,11 @@ require ( github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834 github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-cidutil v0.0.2 - github.com/ipfs/go-datastore v0.4.4 + github.com/ipfs/go-datastore v0.4.5 github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e github.com/ipfs/go-ds-leveldb v0.4.2 github.com/ipfs/go-ds-measure v0.1.0 + github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 github.com/ipfs/go-graphsync v0.2.1 diff --git a/go.sum b/go.sum index 28606d17a..19e33d294 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA= +github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -125,6 +127,14 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b h1:OKALTB609+19AM7wsO0k8yMwAqjEIppcnYvyIhA+ZlQ= +github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ= +github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbLi0587YXpi8tOJT52qCOI/1I0UNThc/I= +github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -225,8 +235,6 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161 github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= -github.com/filecoin-project/go-bitfield v0.2.1-0.20200920171219-7c2059195a8c h1:eEmdVMWo7AngX9fGZSSAm/V6+7tqiBawFfHRjW35JwU= -github.com/filecoin-project/go-bitfield v0.2.1-0.20200920171219-7c2059195a8c/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g= github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -283,6 +291,9 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1/go.mod h1:0eHX/BVySxPc6SE2mZRoppGq7qcEagxdmQnA3dzork8= +github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -351,6 +362,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -491,6 +504,8 @@ github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13X github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg= +github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -509,6 +524,8 @@ github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9 github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-measure v0.1.0 h1:vE4TyY4aeLeVgnnPBC5QzKIjKrqzha0NCujTfgvVbVQ= github.com/ipfs/go-ds-measure v0.1.0/go.mod h1:1nDiFrhLlwArTME1Ees2XaBOl49OoCgd2A3f8EchMSY= +github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61 h1:2wNNdpETSZgnsgy7wx7O6ueu+LCSZRedWrAsIPiOeFE= +github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61/go.mod h1:oh4liWHulKcDKVhCska5NLelE3MatWl+1FwSz3tY91g= github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEPb0= github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM= github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= @@ -1521,6 +1538,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROE golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From 55c6b88537dd9f7b5db9aa601127634b9d13c96b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 22 Sep 2020 00:46:31 +0200 Subject: [PATCH 62/80] Add toggle for badger, flag out gas tracing Signed-off-by: Jakub Sztandera --- chain/vm/runtime.go | 56 ++++++++++++++++++++----------------- chain/vm/vm.go | 19 +++++++++---- cmd/lotus-bench/import.go | 58 +++++++++++++++++++++------------------ 3 files changed, 75 insertions(+), 58 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index eb4476718..72dd413ed 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -459,8 +459,10 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { } func (rt *Runtime) finilizeGasTracing() { - if rt.lastGasCharge != nil { - rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) + if enableTracing { + if rt.lastGasCharge != nil { + rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) + } } } @@ -489,35 +491,39 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) { } +var enableTracing = false + func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError { toUse := gas.Total() - var callers [10]uintptr + if enableTracing { + var callers [10]uintptr - cout := 0 //gruntime.Callers(2+skip, callers[:]) + cout := 0 //gruntime.Callers(2+skip, callers[:]) - now := build.Clock.Now() - if rt.lastGasCharge != nil { - rt.lastGasCharge.TimeTaken = now.Sub(rt.lastGasChargeTime) + now := build.Clock.Now() + if rt.lastGasCharge != nil { + rt.lastGasCharge.TimeTaken = now.Sub(rt.lastGasChargeTime) + } + + gasTrace := types.GasTrace{ + Name: gas.Name, + Extra: gas.Extra, + + TotalGas: toUse, + ComputeGas: gas.ComputeGas, + StorageGas: gas.StorageGas, + + TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti, + VirtualComputeGas: gas.VirtualCompute, + VirtualStorageGas: gas.VirtualStorage, + + Callers: callers[:cout], + } + rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) + rt.lastGasChargeTime = now + rt.lastGasCharge = &gasTrace } - gasTrace := types.GasTrace{ - Name: gas.Name, - Extra: gas.Extra, - - TotalGas: toUse, - ComputeGas: gas.ComputeGas, - StorageGas: gas.StorageGas, - - TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti, - VirtualComputeGas: gas.VirtualCompute, - VirtualStorageGas: gas.VirtualStorage, - - Callers: callers[:cout], - } - rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) - rt.lastGasChargeTime = now - rt.lastGasCharge = &gasTrace - // overflow safe if rt.gasUsed > rt.gasAvailable-toUse { rt.gasUsed = rt.gasAvailable diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 54ea47698..c566ec1eb 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -227,14 +227,21 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, } rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) - rt.lastGasChargeTime = start + if enableTracing { + rt.lastGasChargeTime = start + if parent != nil { + rt.lastGasChargeTime = parent.lastGasChargeTime + rt.lastGasCharge = parent.lastGasCharge + defer func() { + parent.lastGasChargeTime = rt.lastGasChargeTime + parent.lastGasCharge = rt.lastGasCharge + }() + } + } + if parent != nil { - rt.lastGasChargeTime = parent.lastGasChargeTime - rt.lastGasCharge = parent.lastGasCharge defer func() { - parent.gasUsed = rt.gasUsed - parent.lastGasChargeTime = rt.lastGasChargeTime - parent.lastGasCharge = rt.lastGasCharge + parent.gasUsed += rt.gasUsed }() } if gasCharge != nil { diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index b535ed96c..c3554f939 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -119,33 +119,37 @@ var importBenchCmd = &cli.Command{ tdir = tmp } - bdgOpt := badger.DefaultOptions - bdgOpt.GcInterval = 0 - bdgOpt.Options = bdg.DefaultOptions("") - bdgOpt.Options.SyncWrites = false - bdgOpt.Options.Truncate = true - bdgOpt.Options.DetectConflicts = false - - cache := 512 - bds, err := pebbleds.NewDatastore(tdir, &pebble.Options{ - // Pebble has a single combined cache area and the write - // buffers are taken from this too. Assign all available - // memory allowance for cache. - Cache: pebble.NewCache(int64(cache * 1024 * 1024)), - // The size of memory table(as well as the write buffer). - // Note, there may have more than two memory tables in the system. - // MemTableStopWritesThreshold can be configured to avoid the memory abuse. - MemTableSize: cache * 1024 * 1024 / 4, - // The default compaction concurrency(1 thread), - // Here use all available CPUs for faster compaction. - MaxConcurrentCompactions: runtime.NumCPU(), - // Per-level options. Options for at least one level must be specified. The - // options for the last level are used for all subsequent levels. - Levels: []pebble.LevelOptions{ - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - }, - Logger: log, - }) + var bds datastore.Batching + if false { + cache := 512 + bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{ + // Pebble has a single combined cache area and the write + // buffers are taken from this too. Assign all available + // memory allowance for cache. + Cache: pebble.NewCache(int64(cache * 1024 * 1024)), + // The size of memory table(as well as the write buffer). + // Note, there may have more than two memory tables in the system. + // MemTableStopWritesThreshold can be configured to avoid the memory abuse. + MemTableSize: cache * 1024 * 1024 / 4, + // The default compaction concurrency(1 thread), + // Here use all available CPUs for faster compaction. + MaxConcurrentCompactions: runtime.NumCPU(), + // Per-level options. Options for at least one level must be specified. The + // options for the last level are used for all subsequent levels. + Levels: []pebble.LevelOptions{ + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + }, + Logger: log, + }) + } else { + bdgOpt := badger.DefaultOptions + bdgOpt.GcInterval = 0 + bdgOpt.Options = bdg.DefaultOptions("") + bdgOpt.Options.SyncWrites = false + bdgOpt.Options.Truncate = true + bdgOpt.Options.DetectConflicts = false + bds, err = badger.NewDatastore(tdir, &bdgOpt) + } if err != nil { return err } From ff8c0af8c82c022b86fd7f1043460711c31a6c43 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 22 Sep 2020 00:54:11 +0200 Subject: [PATCH 63/80] Add only-import option Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index c3554f939..6af1bfc02 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -90,6 +90,9 @@ var importBenchCmd = &cli.Command{ &cli.Int64Flag{ Name: "start-at", }, + &cli.BoolFlag{ + Name: "only-import", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -119,6 +122,13 @@ var importBenchCmd = &cli.Command{ tdir = tmp } + bdgOpt := badger.DefaultOptions + bdgOpt.GcInterval = 0 + bdgOpt.Options = bdg.DefaultOptions("") + bdgOpt.Options.SyncWrites = false + bdgOpt.Options.Truncate = true + bdgOpt.Options.DetectConflicts = false + var bds datastore.Batching if false { cache := 512 @@ -142,17 +152,12 @@ var importBenchCmd = &cli.Command{ Logger: log, }) } else { - bdgOpt := badger.DefaultOptions - bdgOpt.GcInterval = 0 - bdgOpt.Options = bdg.DefaultOptions("") - bdgOpt.Options.SyncWrites = false - bdgOpt.Options.Truncate = true - bdgOpt.Options.DetectConflicts = false bds, err = badger.NewDatastore(tdir, &bdgOpt) } if err != nil { return err } + defer bds.Close() if cctx.Bool("only-gc") { log.Info("calling CollectGarbage on main ds") @@ -176,6 +181,7 @@ var importBenchCmd = &cli.Command{ if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) } + defer scds.Close() if cctx.Bool("only-gc") { log.Info("calling CollectGarbage on syscall ds") @@ -223,6 +229,10 @@ var importBenchCmd = &cli.Command{ } } + if cctx.Bool("only-import") { + return nil + } + gb, err := cs.GetTipsetByHeight(context.TODO(), 0, head, true) if err != nil { return err From 76db65b1afcfa3f752ff51afcdc9021bacd38415 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 22 Sep 2020 00:57:37 +0200 Subject: [PATCH 64/80] Update pebble Signed-off-by: Jakub Sztandera --- chain/vm/vm.go | 2 +- cmd/lotus-bench/import.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index c566ec1eb..44979454f 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -241,7 +241,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, if parent != nil { defer func() { - parent.gasUsed += rt.gasUsed + parent.gasUsed = rt.gasUsed }() } if gasCharge != nil { diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 6af1bfc02..94dae0d98 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -130,7 +130,7 @@ var importBenchCmd = &cli.Command{ bdgOpt.Options.DetectConflicts = false var bds datastore.Batching - if false { + if true { cache := 512 bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{ // Pebble has a single combined cache area and the write @@ -147,7 +147,7 @@ var importBenchCmd = &cli.Command{ // Per-level options. Options for at least one level must be specified. The // options for the last level are used for all subsequent levels. Levels: []pebble.LevelOptions{ - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + {TargetFileSize: 16 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10), Compression: pebble.NoCompression}, }, Logger: log, }) diff --git a/go.mod b/go.mod index 3b3b4fed7..17cb2fd36 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e github.com/ipfs/go-ds-leveldb v0.4.2 github.com/ipfs/go-ds-measure v0.1.0 - github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61 + github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 github.com/ipfs/go-graphsync v0.2.1 diff --git a/go.sum b/go.sum index 19e33d294..6929c9a23 100644 --- a/go.sum +++ b/go.sum @@ -524,8 +524,8 @@ github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9 github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-measure v0.1.0 h1:vE4TyY4aeLeVgnnPBC5QzKIjKrqzha0NCujTfgvVbVQ= github.com/ipfs/go-ds-measure v0.1.0/go.mod h1:1nDiFrhLlwArTME1Ees2XaBOl49OoCgd2A3f8EchMSY= -github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61 h1:2wNNdpETSZgnsgy7wx7O6ueu+LCSZRedWrAsIPiOeFE= -github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61/go.mod h1:oh4liWHulKcDKVhCska5NLelE3MatWl+1FwSz3tY91g= +github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 h1:W3YMLEvOXqdW+sYMiguhWP6txJwQvIQqhvpU8yAMGQs= +github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459/go.mod h1:oh4liWHulKcDKVhCska5NLelE3MatWl+1FwSz3tY91g= github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEPb0= github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM= github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= From 0d914ac1d4c4d0e8955c7074e9dd03eb49f44900 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 22 Sep 2020 22:01:41 +0200 Subject: [PATCH 65/80] Switch to badger Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 94dae0d98..c8328684e 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -130,7 +130,7 @@ var importBenchCmd = &cli.Command{ bdgOpt.Options.DetectConflicts = false var bds datastore.Batching - if true { + if false { cache := 512 bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{ // Pebble has a single combined cache area and the write From 7e8c6e507055f1f75607facbc0d0220608540343 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 23 Sep 2020 18:44:41 +0200 Subject: [PATCH 66/80] Remove statediff, fix lint, go mod tidy Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 20 +++----------------- go.mod | 7 ++++--- go.sum | 23 ----------------------- 3 files changed, 7 insertions(+), 43 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index c8328684e..3d93b0e5e 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -30,7 +30,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/statediff" bdg "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" @@ -80,9 +79,6 @@ var importBenchCmd = &cli.Command{ Name: "no-import", Usage: "should we import the chain? if set to true chain has to be previously imported", }, - &cli.BoolFlag{ - Name: "only-gc", - }, &cli.BoolFlag{ Name: "global-profile", Value: true, @@ -157,13 +153,8 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } - defer bds.Close() + defer bds.Close() //nolint:errcheck - if cctx.Bool("only-gc") { - log.Info("calling CollectGarbage on main ds") - //bds.CollectGarbage() - log.Info("done calling CollectGarbage on main ds") - } bs := blockstore.NewBlockstore(bds) cacheOpts := blockstore.DefaultCacheOpts() cacheOpts.HasBloomFilterSize = 0 @@ -181,13 +172,8 @@ var importBenchCmd = &cli.Command{ if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) } - defer scds.Close() + defer scds.Close() //nolint:errcheck - if cctx.Bool("only-gc") { - log.Info("calling CollectGarbage on syscall ds") - scds.CollectGarbage() - log.Info("done calling CollectGarbage on syscall ds") - } verifier = &cachingVerifier{ ds: scds, backend: verifier, @@ -317,7 +303,7 @@ var importBenchCmd = &cli.Command{ } fmt.Println("TRACE") fmt.Println(string(d)) - fmt.Println(statediff.Diff(context.Background(), bs, tschain[i-1].ParentState(), st, statediff.ExpandActors)) + //fmt.Println(statediff.Diff(context.Background(), bs, tschain[i-1].ParentState(), st, statediff.ExpandActors)) return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), st) } } diff --git a/go.mod b/go.mod index 17cb2fd36..b8896d78f 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/GeertJohan/go.rice v1.0.0 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b @@ -38,10 +39,10 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/statediff v0.0.6-0.20200918150628-da86dd0d264c github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 + github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 @@ -118,6 +119,7 @@ require ( github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 go.opencensus.io v0.22.4 + go.uber.org/dig v1.10.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 @@ -127,12 +129,11 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 gopkg.in/cheggaaa/pb.v1 v1.0.28 gotest.tools v2.2.0+incompatible + launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) replace github.com/filecoin-project/lotus => ./ -replace github.com/filecoin-project/statediff => ./../statediff - replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi diff --git a/go.sum b/go.sum index 6929c9a23..cbceb52c6 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= -github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= -github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= @@ -218,7 +216,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanw/esbuild v0.6.28/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -231,8 +228,6 @@ github.com/filecoin-project/go-address v0.0.4 h1:gSNMv0qWwH16fGQs7ycOUrDjY6YCSsg github.com/filecoin-project/go-address v0.0.4/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= -github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161 h1:K6t4Hrs+rwUxBz2xg88Bdqeh4k5/rycQFdPseZhRyfE= -github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g= @@ -720,7 +715,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= @@ -1048,8 +1042,6 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1352,8 +1344,6 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1406,7 +1396,6 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20200723185710-6a3894a6352b/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee h1:U7zWWvvAjT76EiuWPSOiZlQDnaQYPxPoxugTtTAcJK0= @@ -1433,8 +1422,6 @@ github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d/go.mod h1:g7c github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= -github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345 h1:IJVAwIctqDFOrO0C2qzksXmANviyHJzrklU27e1ltzE= -github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345/go.mod h1:D7hA8H5pyQx7Y5Em7IWx1R4vNJzfon3gpG9nxjkITjQ= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829 h1:wb7xrDzfkLgPHsSEBm+VSx6aDdi64VtV0xvP0E6j8bk= @@ -1523,7 +1510,6 @@ golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1536,8 +1522,6 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -1553,8 +1537,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1672,7 +1654,6 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1681,7 +1662,6 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= @@ -1725,7 +1705,6 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1854,8 +1833,6 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= From 247a5e2c496eaae153cab85d88335db922e36ed0 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 24 Sep 2020 17:49:58 +0200 Subject: [PATCH 67/80] Go mod tidy Signed-off-by: Jakub Sztandera --- go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go.sum b/go.sum index cbceb52c6..0f309ce1a 100644 --- a/go.sum +++ b/go.sum @@ -528,10 +528,6 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= -github.com/ipfs/go-graphsync v0.2.0 h1:x94MvHLNuRwBlZzVal7tR1RYK7T7H6bqQLPopxDbIF0= -github.com/ipfs/go-graphsync v0.2.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= -github.com/ipfs/go-graphsync v0.1.2 h1:25Ll9kIXCE+DY0dicvfS3KMw+U5sd01b/FJbA7KAbhg= -github.com/ipfs/go-graphsync v0.1.2/go.mod h1:sLXVXm1OxtE2XYPw62MuXCdAuNwkAdsbnfrmos5odbA= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= From baef3c8dd26318fb435c894bfd030e7daf68834a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Sep 2020 15:22:46 +0200 Subject: [PATCH 68/80] sectorstorage: Fix potential panic in FinalizeSector --- .../sector-storage/ffiwrapper/sealer_cgo.go | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index 9bc2680ed..d75501838 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -546,34 +546,37 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID, keepU defer done() pf, err := openPartialFile(maxPieceSize, paths.Unsealed) - if xerrors.Is(err, os.ErrNotExist) { - return xerrors.Errorf("opening partial file: %w", err) - } + if err == nil { + var at uint64 + for sr.HasNext() { + r, err := sr.NextRun() + if err != nil { + _ = pf.Close() + return err + } - var at uint64 - for sr.HasNext() { - r, err := sr.NextRun() - if err != nil { - _ = pf.Close() + offset := at + at += r.Len + if !r.Val { + continue + } + + err = pf.Free(storiface.PaddedByteIndex(abi.UnpaddedPieceSize(offset).Padded()), abi.UnpaddedPieceSize(r.Len).Padded()) + if err != nil { + _ = pf.Close() + return xerrors.Errorf("free partial file range: %w", err) + } + } + + if err := pf.Close(); err != nil { return err } - - offset := at - at += r.Len - if !r.Val { - continue - } - - err = pf.Free(storiface.PaddedByteIndex(abi.UnpaddedPieceSize(offset).Padded()), abi.UnpaddedPieceSize(r.Len).Padded()) - if err != nil { - _ = pf.Close() - return xerrors.Errorf("free partial file range: %w", err) + } else { + if !xerrors.Is(err, os.ErrNotExist) { + return xerrors.Errorf("opening partial file: %w", err) } } - if err := pf.Close(); err != nil { - return err - } } paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache, 0, stores.PathStorage) From 9fe32b7777af8037e1572ccb84feb639970b049c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 01:10:03 -0400 Subject: [PATCH 69/80] Fix wallet list --- cli/wallet.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cli/wallet.go b/cli/wallet.go index 0d69673f9..aa5b9bed3 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -89,10 +89,8 @@ var walletList = &cli.Command{ return err } - def, err := api.WalletDefaultAddress(ctx) - if err != nil { - return err - } + // Assume an error means no default key is set + def, _ := api.WalletDefaultAddress(ctx) tw := tablewriter.New( tablewriter.Col("Address"), From a4e71174297138fdb4d25240fafe4b90abfc656d Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 27 Sep 2020 17:52:26 -0400 Subject: [PATCH 70/80] Add lotus shed util to validate a tipset --- api/api_full.go | 3 ++ api/apistruct/struct.go | 5 ++++ chain/store/store.go | 10 +++++++ cmd/lotus-shed/main.go | 1 + cmd/lotus-shed/sync.go | 64 +++++++++++++++++++++++++++++++++++++++++ node/impl/full/sync.go | 26 +++++++++++++++++ 6 files changed, 109 insertions(+) create mode 100644 cmd/lotus-shed/sync.go diff --git a/api/api_full.go b/api/api_full.go index 6d2d0c7b5..0e5622f4c 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -176,6 +176,9 @@ type FullNode interface { // the reason. SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) + // SyncValidateTipset indicates whether the provided tipset is valid or not + SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) + // MethodGroup: Mpool // The Mpool methods are for interacting with the message pool. The message pool // manages all incoming and outgoing 'messages' going over the network. diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index d5b6950ad..cc2b8b5b5 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -112,6 +112,7 @@ type FullNodeStruct struct { SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` SyncUnmarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` SyncCheckBad func(ctx context.Context, bcid cid.Cid) (string, error) `perm:"read"` + SyncValidateTipset func(ctx context.Context, tsk types.TipSetKey) (bool, error) `perm:"read"` MpoolGetConfig func(context.Context) (*types.MpoolConfig, error) `perm:"read"` MpoolSetConfig func(context.Context, *types.MpoolConfig) error `perm:"write"` @@ -735,6 +736,10 @@ func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string return c.Internal.SyncCheckBad(ctx, bcid) } +func (c *FullNodeStruct) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) { + return c.Internal.SyncValidateTipset(ctx, tsk) +} + func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) { return c.Internal.StateNetworkName(ctx) } diff --git a/chain/store/store.go b/chain/store/store.go index 6c93db7a0..0806fb921 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -286,6 +286,16 @@ func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) e return nil } +func (cs *ChainStore) UnmarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error { + key := blockValidationCacheKeyPrefix.Instance(blkid.String()) + + if err := cs.ds.Delete(key); err != nil { + return xerrors.Errorf("removing from valid block cache: %w", err) + } + + return nil +} + func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error { ts, err := types.NewTipSet([]*types.BlockHeader{b}) if err != nil { diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index c7ded7a25..3864d3014 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -38,6 +38,7 @@ func main() { exportChainCmd, consensusCmd, serveDealStatsCmd, + syncCmd, } app := &cli.App{ diff --git a/cmd/lotus-shed/sync.go b/cmd/lotus-shed/sync.go new file mode 100644 index 000000000..bfe7cc8b7 --- /dev/null +++ b/cmd/lotus-shed/sync.go @@ -0,0 +1,64 @@ +package main + +import ( + "fmt" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/urfave/cli/v2" +) + +var syncCmd = &cli.Command{ + Name: "sync", + Usage: "tools for diagnosing sync issues", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + syncValidateCmd, + }, +} + +var syncValidateCmd = &cli.Command{ + Name: "validate", + Usage: "checks whether a provided tipset is valid", + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + + defer closer() + ctx := lcli.ReqContext(cctx) + + if cctx.Args().Len() < 1 { + fmt.Println("usage: ...") + fmt.Println("At least one block cid must be provided") + return nil + } + + args := cctx.Args().Slice() + + var tscids []cid.Cid + for _, s := range args { + c, err := cid.Decode(s) + if err != nil { + return fmt.Errorf("block cid was invalid: %s", err) + } + tscids = append(tscids, c) + } + + tsk := types.NewTipSetKey(tscids...) + + valid, err := api.SyncValidateTipset(ctx, tsk) + if err != nil { + fmt.Println("Tipset is invalid: ", err) + } + + if valid { + fmt.Println("Tipset is valid") + } + + return nil + }, +} diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index dc3bfe230..221942673 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -126,3 +126,29 @@ func (a *SyncAPI) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error return reason, nil } + +func (a *SyncAPI) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) { + ts, err := a.Syncer.ChainStore().LoadTipSet(tsk) + if err != nil { + return false, err + } + + fts, err := a.Syncer.ChainStore().TryFillTipSet(ts) + if err != nil { + return false, err + } + + for _, blk := range tsk.Cids() { + err = a.Syncer.ChainStore().UnmarkBlockAsValidated(ctx, blk) + if err != nil { + return false, err + } + } + + err = a.Syncer.ValidateTipSet(ctx, fts) + if err != nil { + return false, err + } + + return true, nil +} From 73d193bd9cd1b4d5e9355058febed293087a9364 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 27 Sep 2020 18:08:58 -0400 Subject: [PATCH 71/80] Update docs --- documentation/en/api-methods.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ed082ccbf..29271bdd5 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -169,6 +169,7 @@ * [SyncState](#SyncState) * [SyncSubmitBlock](#SyncSubmitBlock) * [SyncUnmarkBad](#SyncUnmarkBad) + * [SyncValidateTipset](#SyncValidateTipset) * [Wallet](#Wallet) * [WalletBalance](#WalletBalance) * [WalletDefaultAddress](#WalletDefaultAddress) @@ -4379,6 +4380,28 @@ Inputs: Response: `{}` +### SyncValidateTipset +SyncValidateTipset indicates whether the provided tipset is valid or not + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `true` + ## Wallet From c45c8f34a16605a087bd4c6b5b022f8828784156 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 01:39:06 -0400 Subject: [PATCH 72/80] Parametrise whether sync validators should use cache --- chain/sync.go | 28 ++++++++++++++++------------ chain/sync_test.go | 4 ++-- node/impl/full/sync.go | 9 +-------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 78e5178d1..b2e3bb7f1 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -597,7 +597,7 @@ func isPermanent(err error) bool { return !errors.Is(err, ErrTemporal) } -func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) error { +func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet, useCache bool) error { ctx, span := trace.StartSpan(ctx, "validateTipSet") defer span.End() @@ -613,7 +613,7 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) b := b // rebind to a scoped variable futures = append(futures, async.Err(func() error { - if err := syncer.ValidateBlock(ctx, b); err != nil { + if err := syncer.ValidateBlock(ctx, b, useCache); err != nil { if isPermanent(err) { syncer.bad.Add(b.Cid(), NewBadBlockReason([]cid.Cid{b.Cid()}, err.Error())) } @@ -680,7 +680,7 @@ func blockSanityChecks(h *types.BlockHeader) error { } // ValidateBlock should match up with 'Semantical Validation' in validation.md in the spec -func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (err error) { +func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, useCache bool) (err error) { defer func() { // b.Cid() could panic for empty blocks that are used in tests. if rerr := recover(); rerr != nil { @@ -689,13 +689,15 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er } }() - isValidated, err := syncer.store.IsBlockValidated(ctx, b.Cid()) - if err != nil { - return xerrors.Errorf("check block validation cache %s: %w", b.Cid(), err) - } + if useCache { + isValidated, err := syncer.store.IsBlockValidated(ctx, b.Cid()) + if err != nil { + return xerrors.Errorf("check block validation cache %s: %w", b.Cid(), err) + } - if isValidated { - return nil + if isValidated { + return nil + } } validationStart := build.Clock.Now() @@ -959,8 +961,10 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er return mulErr } - if err := syncer.store.MarkBlockAsValidated(ctx, b.Cid()); err != nil { - return xerrors.Errorf("caching block validation %s: %w", b.Cid(), err) + if useCache { + if err := syncer.store.MarkBlockAsValidated(ctx, b.Cid()); err != nil { + return xerrors.Errorf("caching block validation %s: %w", b.Cid(), err) + } } return nil @@ -1462,7 +1466,7 @@ func (syncer *Syncer) syncMessagesAndCheckState(ctx context.Context, headers []* return syncer.iterFullTipsets(ctx, headers, func(ctx context.Context, fts *store.FullTipSet) error { log.Debugw("validating tipset", "height", fts.TipSet().Height(), "size", len(fts.TipSet().Cids())) - if err := syncer.ValidateTipSet(ctx, fts); err != nil { + if err := syncer.ValidateTipSet(ctx, fts, true); err != nil { log.Errorf("failed to validate tipset: %+v", err) return xerrors.Errorf("message processing failed: %w", err) } diff --git a/chain/sync_test.go b/chain/sync_test.go index 7a839be2b..1b06f604b 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -732,7 +732,7 @@ func TestSyncInputs(t *testing.T) { err := s.ValidateBlock(context.TODO(), &types.FullBlock{ Header: &types.BlockHeader{}, - }) + }, false) if err == nil { t.Fatal("should error on empty block") } @@ -741,7 +741,7 @@ func TestSyncInputs(t *testing.T) { h.ElectionProof = nil - err = s.ValidateBlock(context.TODO(), &types.FullBlock{Header: h}) + err = s.ValidateBlock(context.TODO(), &types.FullBlock{Header: h}, false) if err == nil { t.Fatal("should error on block with nil election proof") } diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index 221942673..1bd3af415 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -138,14 +138,7 @@ func (a *SyncAPI) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) ( return false, err } - for _, blk := range tsk.Cids() { - err = a.Syncer.ChainStore().UnmarkBlockAsValidated(ctx, blk) - if err != nil { - return false, err - } - } - - err = a.Syncer.ValidateTipSet(ctx, fts) + err = a.Syncer.ValidateTipSet(ctx, fts, false) if err != nil { return false, err } From a388bcfad61d90dc5718982095c1707f6d858cdf Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 01:43:10 -0400 Subject: [PATCH 73/80] Add an endpoint to validate whether a string is a well-formed address --- api/api_full.go | 2 ++ api/apistruct/struct.go | 29 +++++++++++++++++------------ node/impl/full/wallet.go | 4 ++++ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 6d2d0c7b5..7b7574c26 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -244,6 +244,8 @@ type FullNode interface { WalletImport(context.Context, *types.KeyInfo) (address.Address, error) // WalletDelete deletes an address from the wallet. WalletDelete(context.Context, address.Address) error + // WalletValidateAddress validates whether a given string can be decoded as a well-formed address + WalletValidateAddress(context.Context, string) (address.Address, error) // Other diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index d5b6950ad..73c4d8ed5 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -129,18 +129,19 @@ type FullNodeStruct struct { MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"` MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error) `perm:"write"` - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` - WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` - WalletList func(context.Context) ([]address.Address, error) `perm:"write"` - WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"` - WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` - WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` - WalletVerify func(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) `perm:"read"` - WalletDefaultAddress func(context.Context) (address.Address, error) `perm:"write"` - WalletSetDefault func(context.Context, address.Address) error `perm:"admin"` - WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` - WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` - WalletDelete func(context.Context, address.Address) error `perm:"write"` + WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` + WalletList func(context.Context) ([]address.Address, error) `perm:"write"` + WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"` + WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` + WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` + WalletVerify func(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) `perm:"read"` + WalletDefaultAddress func(context.Context) (address.Address, error) `perm:"write"` + WalletSetDefault func(context.Context, address.Address) error `perm:"admin"` + WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` + WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` + WalletDelete func(context.Context, address.Address) error `perm:"write"` + WalletValidateAddress func(context.Context, string) (address.Address, error) `perm:"read"` ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"` ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` @@ -631,6 +632,10 @@ func (c *FullNodeStruct) WalletDelete(ctx context.Context, addr address.Address) return c.Internal.WalletDelete(ctx, addr) } +func (c *FullNodeStruct) WalletValidateAddress(ctx context.Context, str string) (address.Address, error) { + return c.Internal.WalletValidateAddress(ctx, str) +} + func (c *FullNodeStruct) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) { return c.Internal.MpoolGetNonce(ctx, addr) } diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 64231b74e..b2ecdebbd 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -90,3 +90,7 @@ func (a *WalletAPI) WalletImport(ctx context.Context, ki *types.KeyInfo) (addres func (a *WalletAPI) WalletDelete(ctx context.Context, addr address.Address) error { return a.Wallet.DeleteKey(addr) } + +func (a *WalletAPI) WalletValidateAddress(ctx context.Context, str string) (address.Address, error) { + return address.NewFromString(str) +} From bc4cbdc8957e45daca3b24ed5f434ebf72ced58b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 01:45:03 -0400 Subject: [PATCH 74/80] Update docs --- documentation/en/api-methods.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ed082ccbf..beaf7f3b6 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -181,6 +181,7 @@ * [WalletSetDefault](#WalletSetDefault) * [WalletSign](#WalletSign) * [WalletSignMessage](#WalletSignMessage) + * [WalletValidateAddress](#WalletValidateAddress) * [WalletVerify](#WalletVerify) ## @@ -4585,6 +4586,21 @@ Response: } ``` +### WalletValidateAddress +WalletValidateAddress validates whether a given string can be decoded as a well-formed address + + +Perms: read + +Inputs: +```json +[ + "string value" +] +``` + +Response: `"t01234"` + ### WalletVerify WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid. The address does not have to be in the wallet. From 1affd498c172ac7b29192b09f67be76b83fabe2e Mon Sep 17 00:00:00 2001 From: Dan Shao Date: Wed, 30 Sep 2020 14:23:35 +0800 Subject: [PATCH 75/80] Add --no-swap flag for worker --- cmd/lotus-seal-worker/main.go | 7 +++++++ extern/sector-storage/localworker.go | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index e36514bb8..d2c57e680 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -109,6 +109,11 @@ var runCmd = &cli.Command{ Name: "no-local-storage", Usage: "don't use storageminer repo for sector storage", }, + &cli.BoolFlag{ + Name: "no-swap", + Usage: "don't use swap", + Value: false, + }, &cli.BoolFlag{ Name: "addpiece", Usage: "enable addpiece", @@ -346,6 +351,7 @@ var runCmd = &cli.Command{ LocalWorker: sectorstorage.NewLocalWorker(sectorstorage.WorkerConfig{ SealProof: spt, TaskTypes: taskTypes, + NoSwap: cctx.Bool("no-swap"), }, remote, localStore, nodeApi), localStore: localStore, ls: lr, @@ -465,6 +471,7 @@ func watchMinerConn(ctx context.Context, cctx *cli.Context, nodeApi api.StorageM "run", fmt.Sprintf("--listen=%s", cctx.String("listen")), fmt.Sprintf("--no-local-storage=%t", cctx.Bool("no-local-storage")), + fmt.Sprintf("--no-swap=%t", cctx.Bool("no-swap")), fmt.Sprintf("--addpiece=%t", cctx.Bool("addpiece")), fmt.Sprintf("--precommit1=%t", cctx.Bool("precommit1")), fmt.Sprintf("--unseal=%t", cctx.Bool("unseal")), diff --git a/extern/sector-storage/localworker.go b/extern/sector-storage/localworker.go index 2c3c350f7..b1193a2e2 100644 --- a/extern/sector-storage/localworker.go +++ b/extern/sector-storage/localworker.go @@ -26,6 +26,7 @@ var pathTypes = []stores.SectorFileType{stores.FTUnsealed, stores.FTSealed, stor type WorkerConfig struct { SealProof abi.RegisteredSealProof TaskTypes []sealtasks.TaskType + NoSwap bool } type LocalWorker struct { @@ -33,6 +34,7 @@ type LocalWorker struct { storage stores.Store localStore *stores.Local sindex stores.SectorIndex + noSwap bool acceptTasks map[sealtasks.TaskType]struct{} } @@ -50,6 +52,7 @@ func NewLocalWorker(wcfg WorkerConfig, store stores.Store, local *stores.Local, storage: store, localStore: local, sindex: sindex, + noSwap: wcfg.NoSwap, acceptTasks: acceptTasks, } @@ -275,11 +278,16 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err) } + memSwap := mem.VirtualTotal + if l.noSwap { + memSwap = 0 + } + return storiface.WorkerInfo{ Hostname: hostname, Resources: storiface.WorkerResources{ MemPhysical: mem.Total, - MemSwap: mem.VirtualTotal, + MemSwap: memSwap, MemReserved: mem.VirtualUsed + mem.Total - mem.Available, // TODO: sub this process CPUs: uint64(runtime.NumCPU()), GPUs: gpus, From 6abccc4d5ed9c977916a812b3aa3e4407fd124ae Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 02:56:38 -0400 Subject: [PATCH 76/80] Add an option to set config --- cmd/lotus/daemon.go | 8 ++++++++ node/repo/fsrepo.go | 34 ++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index b976fde79..a0f754a60 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -127,6 +127,10 @@ var DaemonCmd = &cli.Command{ Usage: "manage open file limit", Value: true, }, + &cli.StringFlag{ + Name: "config", + Usage: "specify path of config file to use", + }, }, Action: func(cctx *cli.Context) error { err := runmetrics.Enable(runmetrics.RunMetricOptions{ @@ -180,6 +184,10 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("opening fs repo: %w", err) } + if cctx.String("config") != "" { + r.SetConfigPath(cctx.String("config")) + } + if err := r.Init(repo.FullNode); err != nil && err != repo.ErrRepoExists { return xerrors.Errorf("repo init error: %w", err) } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 709d78d3a..a69cdd55d 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -65,7 +65,8 @@ var ErrRepoExists = xerrors.New("repo exists") // FsRepo is struct for repo, use NewFS to create type FsRepo struct { - path string + path string + configPath string } var _ Repo = &FsRepo{} @@ -78,10 +79,15 @@ func NewFS(path string) (*FsRepo, error) { } return &FsRepo{ - path: path, + path: path, + configPath: filepath.Join(path, fsConfig), }, nil } +func (fsr *FsRepo) SetConfigPath(cfgPath string) { + fsr.configPath = cfgPath +} + func (fsr *FsRepo) Exists() (bool, error) { _, err := os.Stat(filepath.Join(fsr.path, fsDatastore)) notexist := os.IsNotExist(err) @@ -115,9 +121,7 @@ func (fsr *FsRepo) Init(t RepoType) error { } func (fsr *FsRepo) initConfig(t RepoType) error { - cfgP := filepath.Join(fsr.path, fsConfig) - - _, err := os.Stat(cfgP) + _, err := os.Stat(fsr.configPath) if err == nil { // exists return nil @@ -125,7 +129,7 @@ func (fsr *FsRepo) initConfig(t RepoType) error { return err } - c, err := os.Create(cfgP) + c, err := os.Create(fsr.configPath) if err != nil { return err } @@ -215,16 +219,18 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) { return nil, xerrors.Errorf("could not lock the repo: %w", err) } return &fsLockedRepo{ - path: fsr.path, - repoType: repoType, - closer: closer, + path: fsr.path, + configPath: fsr.configPath, + repoType: repoType, + closer: closer, }, nil } type fsLockedRepo struct { - path string - repoType RepoType - closer io.Closer + path string + configPath string + repoType RepoType + closer io.Closer ds map[string]datastore.Batching dsErr error @@ -277,7 +283,7 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) { } func (fsr *fsLockedRepo) loadConfigFromDisk() (interface{}, error) { - return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType)) + return config.FromFile(fsr.configPath, defConfForType(fsr.repoType)) } func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error { @@ -306,7 +312,7 @@ func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error { } // write buffer of TOML bytes to config file - err = ioutil.WriteFile(fsr.join(fsConfig), buf.Bytes(), 0644) + err = ioutil.WriteFile(fsr.configPath, buf.Bytes(), 0644) if err != nil { return err } From eb6191d0ffd01a7cf7f8544a31acf307b1799fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 30 Sep 2020 11:02:10 +0100 Subject: [PATCH 77/80] tvx: precursor selection modes; canonical message fetching; basefee. --- cmd/tvx/extract.go | 199 +++++++++++++++++++++++----------------- cmd/tvx/extract_many.go | 43 ++++++--- conformance/driver.go | 58 ++++++++---- conformance/runner.go | 19 +++- go.mod | 2 +- go.sum | 4 +- 6 files changed, 204 insertions(+), 121 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 0dc7f6aa0..fef245858 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -21,21 +21,27 @@ import ( lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/conformance" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/test-vectors/schema" "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" ) +const ( + PrecursorSelectAll = "all" + PrecursorSelectSender = "sender" +) + type extractOpts struct { - id string - block string - class string - cid string - file string - retain string + id string + block string + class string + cid string + file string + retain string + precursor string } var extractFlags extractOpts @@ -81,6 +87,16 @@ var extractCmd = &cli.Command{ Value: "accessed-cids", Destination: &extractFlags.retain, }, + &cli.StringFlag{ + Name: "precursor-select", + Usage: "precursors to apply; values: 'all', 'sender'; 'all' selects all preceding" + + "messages in the canonicalised tipset, 'sender' selects only preceding messages from the same" + + "sender. Usually, 'sender' is a good tradeoff and gives you sufficient accuracy. If the receipt sanity" + + "check fails due to gas reasons, switch to 'all', as previous messages in the tipset may have" + + "affected state in a disruptive way", + Value: "sender", + Destination: &extractFlags.precursor, + }, }, } @@ -124,46 +140,38 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { return fmt.Errorf("failed while fetching circulating supply: %w", err) } - circSupply := circSupplyDetail.FilCirculating.Int64() + circSupply := circSupplyDetail.FilCirculating log.Printf("message was executed in tipset: %s", execTs.Key()) log.Printf("message was included in tipset: %s", incTs.Key()) log.Printf("circulating supply at inclusion tipset: %d", circSupply) - log.Printf("finding precursor messages") + log.Printf("finding precursor messages using mode: %s", opts.precursor) - // Iterate through blocks, finding the one that contains the message and its - // precursors, if any. - var allmsgs []*types.Message - for _, b := range incTs.Blocks() { - messages, err := fapi.ChainGetBlockMessages(ctx, b.Cid()) - if err != nil { - return err - } - - related, found, err := findMsgAndPrecursors(messages, msg) - if err != nil { - return fmt.Errorf("invariant failed while scanning messages in block %s: %w", b.Cid(), err) - } - - if found { - var mcids []cid.Cid - for _, m := range related { - mcids = append(mcids, m.Cid()) - } - log.Printf("found message in block %s; precursors: %v", b.Cid(), mcids[:len(mcids)-1]) - allmsgs = related - break - } - - log.Printf("message not found in block %s; number of precursors found: %d; ignoring block", b.Cid(), len(related)) + // Fetch messages in canonical order from inclusion tipset. + msgs, err := fapi.ChainGetParentMessages(ctx, execTs.Blocks()[0].Cid()) + if err != nil { + return fmt.Errorf("failed to fetch messages in canonical order from inclusion tipset: %w", err) } - if allmsgs == nil { - // Message was not found; abort. - return fmt.Errorf("did not find a block containing the message") + related, found, err := findMsgAndPrecursors(opts.precursor, msg, msgs) + if err != nil { + return fmt.Errorf("failed while finding message and precursors: %w", err) } - precursors := allmsgs[:len(allmsgs)-1] + if !found { + return fmt.Errorf("message not found; precursors found: %d", len(related)) + } + + var ( + precursors = related[:len(related)-1] + precursorsCids []cid.Cid + ) + + for _, p := range precursors { + precursorsCids = append(precursorsCids, p.Cid()) + } + + log.Println(color.GreenString("found message; precursors (count: %d): %v", len(precursors), precursorsCids)) var ( // create a read-through store that uses ChainGetObject to fetch unknown CIDs. @@ -179,11 +187,20 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { root := incTs.ParentState() log.Printf("base state tree root CID: %s", root) + basefee := incTs.Blocks()[0].ParentBaseFee + log.Printf("basefee: %s", basefee) + // on top of that state tree, we apply all precursors. log.Printf("number of precursors to apply: %d", len(precursors)) for i, m := range precursors { log.Printf("applying precursor %d, cid: %s", i, m.Cid()) - _, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m, &circSupplyDetail.FilCirculating) + _, root, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{ + Preroot: root, + Epoch: execTs.Height(), + Message: m, + CircSupply: &circSupplyDetail.FilCirculating, + BaseFee: &basefee, + }) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) } @@ -208,7 +225,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { tbs.StartTracing() preroot = root - applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating) + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{ + Preroot: preroot, + Epoch: execTs.Height(), + Message: msg, + CircSupply: &circSupplyDetail.FilCirculating, + BaseFee: &basefee, + }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) } @@ -233,7 +256,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { if err != nil { return err } - applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating) + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{ + Preroot: preroot, + Epoch: execTs.Height(), + Message: msg, + CircSupply: &circSupplyDetail.FilCirculating, + BaseFee: &basefee, + }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) } @@ -248,21 +277,39 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot) log.Println("performing sanity check on receipt") - receipt := &schema.Receipt{ - ExitCode: int64(applyret.ExitCode), - ReturnValue: applyret.Return, - GasUsed: applyret.GasUsed, + // TODO sometimes this returns a nil receipt and no error ¯\_(ツ)_/¯ + // ex: https://filfox.info/en/message/bafy2bzacebpxw3yiaxzy2bako62akig46x3imji7fewszen6fryiz6nymu2b2 + // This code is lenient and skips receipt comparison in case of a nil receipt. + rec, err := fapi.StateGetReceipt(ctx, mcid, execTs.Key()) + if err != nil { + return fmt.Errorf("failed to find receipt on chain: %w", err) } + log.Printf("found receipt: %+v", rec) - reporter := new(conformance.LogReporter) - conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed") - if reporter.Failed() { - log.Println(color.RedString("receipt sanity check failed; aborting")) - return fmt.Errorf("vector generation aborted") + // generate the schema receipt; if we got + var receipt *schema.Receipt + if rec != nil { + receipt = &schema.Receipt{ + ExitCode: int64(rec.ExitCode), + ReturnValue: rec.Return, + GasUsed: rec.GasUsed, + } + reporter := new(conformance.LogReporter) + conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed") + if reporter.Failed() { + log.Println(color.RedString("receipt sanity check failed; aborting")) + return fmt.Errorf("vector generation aborted") + } + log.Println(color.GreenString("receipt sanity check succeeded")) + } else { + receipt = &schema.Receipt{ + ExitCode: int64(applyret.ExitCode), + ReturnValue: applyret.Return, + GasUsed: applyret.GasUsed, + } + log.Println(color.YellowString("skipping receipts comparison; we got back a nil receipt from lotus")) } - log.Println(color.GreenString("receipt sanity check succeeded")) - log.Println("generating vector") msgBytes, err := msg.Serialize() if err != nil { @@ -312,7 +359,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { CAR: out.Bytes(), Pre: &schema.Preconditions{ Epoch: int64(execTs.Height()), - CircSupply: &circSupply, + CircSupply: circSupply.Int, + BaseFee: basefee.Int, StateTree: &schema.StateTree{ RootCID: preroot, }, @@ -428,41 +476,22 @@ func fetchThisAndPrevTipset(ctx context.Context, api api.FullNode, target types. return targetTs, prevTs, nil } -// findMsgAndPrecursors scans the messages in a block to locate the supplied -// message, looking into the BLS or SECP section depending on the sender's -// address type. -// -// It returns any precursors (if they exist), and the found message (if found), -// in a slice. -// -// It also returns a boolean indicating whether the message was actually found. -// -// This function also asserts invariants, and if those fail, it returns an error. -func findMsgAndPrecursors(messages *api.BlockMessages, target *types.Message) (related []*types.Message, found bool, err error) { - // Decide which block of messages to process, depending on whether the - // sender is a BLS or a SECP account. - input := messages.BlsMessages - if senderKind := target.From.Protocol(); senderKind == address.SECP256K1 { - input = make([]*types.Message, 0, len(messages.SecpkMessages)) - for _, sm := range messages.SecpkMessages { - input = append(input, &sm.Message) - } - } - - for _, other := range input { - if other.From != target.From { - continue - } - - // this message is from the same sender, so it's related. - related = append(related, other) - - if other.Nonce > target.Nonce { - return nil, false, fmt.Errorf("a message with nonce higher than the target was found before the target; offending mcid: %s", other.Cid()) +// findMsgAndPrecursors ranges through the canonical messages slice, locating +// the target message and returning precursors in accordance to the supplied +// mode. +func findMsgAndPrecursors(mode string, target *types.Message, msgs []api.Message) (related []*types.Message, found bool, err error) { + // Range through canonicalised messages, selecting only the precursors based + // on selection mode. + for _, other := range msgs { + switch { + case mode == PrecursorSelectAll: + fallthrough + case mode == PrecursorSelectSender && other.Message.From == target.From: + related = append(related, other.Message) } // this message is the target; we're done. - if other.Cid() == target.Cid() { + if other.Cid == target.Cid() { return related, true, nil } } diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index 83ec72b21..9de7aeab5 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -111,8 +111,12 @@ func runExtractMany(c *cli.Context) error { log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header)) } - var generated []string - merr := new(multierror.Error) + var ( + generated []string + merr = new(multierror.Error) + retry []extractOpts // to retry with 'canonical' precursor selection mode + ) + // Read each row and extract the requested message. for { row, err := reader.Read() @@ -164,19 +168,21 @@ func runExtractMany(c *cli.Context) error { // Vector filename, using a base of outdir. file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json" - log.Println(color.YellowString("processing message id: %s", id)) + log.Println(color.YellowString("processing message cid with 'sender' precursor mode: %s", id)) opts := extractOpts{ - id: id, - block: block, - class: "message", - cid: cid, - file: file, - retain: "accessed-cids", + id: id, + block: block, + class: "message", + cid: cid, + file: file, + retain: "accessed-cids", + precursor: PrecursorSelectSender, } if err := doExtract(ctx, fapi, opts); err != nil { - merr = multierror.Append(err, fmt.Errorf("failed to extract vector for message %s: %w", cid, err)) + log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'canonical' precursor selection", cid, err)) + retry = append(retry, opts) continue } @@ -185,6 +191,21 @@ func runExtractMany(c *cli.Context) error { generated = append(generated, file) } + log.Printf("extractions to try with canonical precursor selection mode: %d", len(retry)) + + for _, r := range retry { + log.Printf("retrying %s: %s", r.cid, r.id) + + r.precursor = PrecursorSelectAll + if err := doExtract(ctx, fapi, r); err != nil { + merr = multierror.Append(merr, fmt.Errorf("failed to extract vector for message %s: %w", r.cid, err)) + continue + } + + log.Println(color.MagentaString("generated file: %s", r.file)) + generated = append(generated, r.file) + } + if len(generated) == 0 { log.Println("no files generated") } else { @@ -195,7 +216,7 @@ func runExtractMany(c *cli.Context) error { } if merr.ErrorOrNil() != nil { - log.Println(color.YellowString("done processing with errors: %s")) + log.Println(color.YellowString("done processing with errors: %s", err)) } else { log.Println(color.GreenString("done processing with no errors")) } diff --git a/conformance/driver.go b/conformance/driver.go index 3f50b67a9..9ced12d74 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -2,6 +2,7 @@ package conformance import ( "context" + "os" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" @@ -23,15 +24,14 @@ import ( ds "github.com/ipfs/go-datastore" ) -// DefaultCirculatingSupply is the fallback circulating supply returned by -// the driver's CircSupplyCalculator function, used if the vector specifies -// no circulating supply. -var DefaultCirculatingSupply = types.TotalFilecoinInt - var ( - // BaseFee to use in the VM. - // TODO make parametrisable through vector. - BaseFee = abi.NewTokenAmount(100) + // DefaultCirculatingSupply is the fallback circulating supply returned by + // the driver's CircSupplyCalculator function, used if the vector specifies + // no circulating supply. + DefaultCirculatingSupply = types.TotalFilecoinInt + + // DefaultBaseFee to use in the VM, if one is not supplied in the vector. + DefaultBaseFee = abi.NewTokenAmount(100) ) type Driver struct { @@ -139,24 +139,46 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot return ret, nil } +type ExecuteMessageParams struct { + Preroot cid.Cid + Epoch abi.ChainEpoch + Message *types.Message + CircSupply *abi.TokenAmount + BaseFee *abi.TokenAmount +} + // ExecuteMessage executes a conformance test vector message in a temporary VM. -func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message, circSupply *abi.TokenAmount) (*vm.ApplyRet, cid.Cid, error) { +func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageParams) (*vm.ApplyRet, cid.Cid, error) { + if !d.vmFlush { + // do not flush the VM, just the state tree; this should be used with + // LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible. + _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") + } + + basefee := DefaultBaseFee + if params.BaseFee != nil { + basefee = *params.BaseFee + } + + circSupply := DefaultCirculatingSupply + if params.CircSupply != nil { + circSupply = *params.CircSupply + } + // dummy state manager; only to reference the GetNetworkVersion method, // which does not depend on state. sm := new(stmgr.StateManager) + vmOpts := &vm.VMOpts{ - StateBase: preroot, - Epoch: epoch, + StateBase: params.Preroot, + Epoch: params.Epoch, Rand: &testRand{}, // TODO always succeeds; need more flexibility. Bstore: bs, Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) { - if circSupply != nil { - return *circSupply, nil - } - return DefaultCirculatingSupply, nil + return circSupply, nil }, - BaseFee: BaseFee, + BaseFee: basefee, NtwkVersion: sm.GetNtwkVersion, } @@ -174,7 +196,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch lvm.SetInvoker(invoker) - ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(msg)) + ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(params.Message)) if err != nil { return nil, cid.Undef, err } @@ -185,8 +207,6 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch // recursive copoy from the temporary blcokstore to the real blockstore. root, err = lvm.Flush(d.ctx) } else { - // do not flush the VM, just the state tree; this should be used with - // LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible. root, err = lvm.StateTree().(*state.StateTree).Flush(d.ctx) } diff --git a/conformance/runner.go b/conformance/runner.go index 812f3cc08..2db53b3e4 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -13,6 +13,7 @@ import ( "github.com/fatih/color" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/test-vectors/schema" "github.com/ipfs/go-blockservice" @@ -43,14 +44,20 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { } // Create a new Driver. - driver := NewDriver(ctx, vector.Selector, DriverOpts{}) + driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true}) var circSupply *abi.TokenAmount if cs := vector.Pre.CircSupply; cs != nil { - ta := abi.NewTokenAmount(*cs) + ta := big.NewFromGo(cs) circSupply = &ta } + var basefee *abi.TokenAmount + if bf := vector.Pre.BaseFee; bf != nil { + ta := big.NewFromGo(bf) + basefee = &ta + } + // Apply every message. for i, m := range vector.ApplyMessages { msg, err := types.DecodeMessage(m.Bytes) @@ -65,7 +72,13 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { // Execute the message. var ret *vm.ApplyRet - ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg, circSupply) + ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{ + Preroot: root, + Epoch: abi.ChainEpoch(epoch), + Message: msg, + CircSupply: circSupply, + BaseFee: basefee, + }) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) } diff --git a/go.mod b/go.mod index bf3748749..1ef228ee7 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.2 + github.com/filecoin-project/test-vectors/schema v0.0.3 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 6a310ed3d..924e8f5c2 100644 --- a/go.sum +++ b/go.sum @@ -258,8 +258,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.2 h1:/Pp//88WBXe0h+ksntdL2HpEgAmbwXrftAfeVG39zdY= -github.com/filecoin-project/test-vectors/schema v0.0.2/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= +github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From f58881e966dbd087172ecd5e1e1958850a3d2455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 30 Sep 2020 14:57:24 +0100 Subject: [PATCH 78/80] minor fixes. --- cmd/tvx/extract_many.go | 4 ++-- cmd/tvx/stores.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index 9de7aeab5..d7a25c10c 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -48,7 +48,7 @@ var extractManyCmd = &cli.Command{ Destination: &extractManyFlags.in, }, &cli.StringFlag{ - Name: "out-dir", + Name: "outdir", Usage: "output directory", Destination: &extractManyFlags.outdir, }, @@ -216,7 +216,7 @@ func runExtractMany(c *cli.Context) error { } if merr.ErrorOrNil() != nil { - log.Println(color.YellowString("done processing with errors: %s", err)) + log.Println(color.YellowString("done processing with errors: %v", merr)) } else { log.Println(color.GreenString("done processing with no errors")) } diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go index 6e50e0839..93e0d215f 100644 --- a/cmd/tvx/stores.go +++ b/cmd/tvx/stores.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/blockstore" - "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/lotus/chain/actors/adt" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" From ff8663faa0b3a7d1871f2c4c681e243c2cbc229d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 30 Sep 2020 14:58:22 +0100 Subject: [PATCH 79/80] update test-vectors submodule. --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 6bea015ed..3a6e0b5e0 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 6bea015edddde116001a4251dce3c4a9966c25d9 +Subproject commit 3a6e0b5e069b1452ce1a032aa315354d645f3ec4 From 1c4f8e83d79d052d1e4408d635daf80b3e81d62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 30 Sep 2020 16:11:44 +0100 Subject: [PATCH 80/80] tvx/extract-many: add batch id, change generated filename. --- cmd/tvx/extract_many.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index d7a25c10c..9679a1dbd 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -20,8 +20,9 @@ import ( ) var extractManyFlags struct { - in string - outdir string + in string + outdir string + batchId string } var extractManyCmd = &cli.Command{ @@ -37,11 +38,17 @@ var extractManyCmd = &cli.Command{ The first row MUST be a header row. At the bare minimum, those seven fields must appear, in the order specified. Extra fields are accepted, but always - after these compulsory six. + after these compulsory seven. `, Action: runExtractMany, Flags: []cli.Flag{ &repoFlag, + &cli.StringFlag{ + Name: "batch-id", + Usage: "batch id; a four-digit left-zero-padded sequential number (e.g. 0041)", + Required: true, + Destination: &extractManyFlags.batchId, + }, &cli.StringFlag{ Name: "in", Usage: "path to input file (csv)", @@ -164,7 +171,7 @@ func runExtractMany(c *cli.Context) error { actorcodename := strings.ReplaceAll(actorcode, "/", "_") // Compute the ID of the vector. - id := fmt.Sprintf("extracted-msg-%s-%s-%s-%s", actorcodename, methodname, exitcodename, seq) + id := fmt.Sprintf("ext-%s-%s-%s-%s-%s", extractManyFlags.batchId, actorcodename, methodname, exitcodename, seq) // Vector filename, using a base of outdir. file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json"