From 779975c3a93ddb43ab0f10fc556d5e696310df3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 25 Apr 2023 09:54:26 +0200 Subject: [PATCH 1/6] ci: Debugging m1 build --- .circleci/config.yml | 13 +++++++------ .circleci/template.yml | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e3eb1c089..0efc8490c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,7 +63,7 @@ commands: name: Install Rust command: | curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: make deps lotus + - run: make deps download-params: steps: - restore_cache: @@ -304,9 +304,7 @@ jobs: darwin: true darwin-architecture: arm64 - run: | - export CPATH=$(brew --prefix)/include - export LIBRARY_PATH=$(brew --prefix)/lib - make lotus lotus-miner lotus-worker + export CPATH=$(brew --prefix)/include && export LIBRARY_PATH=$(brew --prefix)/lib && make lotus lotus-miner lotus-worker - run: otool -hv lotus - run: name: check tag and version output match @@ -1069,6 +1067,7 @@ workflows: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -1078,6 +1077,7 @@ workflows: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -1087,6 +1087,7 @@ workflows: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -1098,8 +1099,8 @@ workflows: - "Build ( darwin / arm64 )" filters: branches: - ignore: - - /.*/ + only: + - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ diff --git a/.circleci/template.yml b/.circleci/template.yml index 382965615..8393eec5b 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -63,7 +63,7 @@ commands: name: Install Rust command: | curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: make deps lotus + - run: make deps download-params: steps: - restore_cache: @@ -304,9 +304,7 @@ jobs: darwin: true darwin-architecture: arm64 - run: | - export CPATH=$(brew --prefix)/include - export LIBRARY_PATH=$(brew --prefix)/lib - make lotus lotus-miner lotus-worker + export CPATH=$(brew --prefix)/include && export LIBRARY_PATH=$(brew --prefix)/lib && make lotus lotus-miner lotus-worker - run: otool -hv lotus - run: name: check tag and version output match @@ -583,6 +581,7 @@ workflows: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -592,6 +591,7 @@ workflows: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -601,6 +601,7 @@ workflows: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -612,8 +613,8 @@ workflows: - "Build ( darwin / arm64 )" filters: branches: - ignore: - - /.*/ + only: + - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ From 9558df7cdaafaab0779a71f55d9b0a4352429b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 25 Apr 2023 10:24:18 +0200 Subject: [PATCH 2/6] temp drop m1 from releases --- .circleci/config.yml | 1 - .circleci/template.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0efc8490c..a94329d81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1096,7 +1096,6 @@ workflows: requires: - "Build ( darwin / amd64 )" - "Build ( linux / amd64 )" - - "Build ( darwin / arm64 )" filters: branches: only: diff --git a/.circleci/template.yml b/.circleci/template.yml index 8393eec5b..ed5ba4850 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -610,7 +610,6 @@ workflows: requires: - "Build ( darwin / amd64 )" - "Build ( linux / amd64 )" - - "Build ( darwin / arm64 )" filters: branches: only: From 0cfb69b95a583138328d18b917acf1de713b2b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 25 Apr 2023 10:25:40 +0200 Subject: [PATCH 3/6] ci: run release on ci/ branches, include m1 --- .circleci/config.yml | 6 ++++-- .circleci/template.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a94329d81..9038fdb8f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1096,10 +1096,11 @@ workflows: requires: - "Build ( darwin / amd64 )" - "Build ( linux / amd64 )" + - "Build ( darwin / arm64 )" filters: branches: - only: - - /^ci\/.*$/ + ignore: + - /^.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -1114,6 +1115,7 @@ workflows: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - /^ci\/.*$/ - build-docker: name: "Docker push (lotus-all-in-one / stable / mainnet)" image: lotus-all-in-one diff --git a/.circleci/template.yml b/.circleci/template.yml index ed5ba4850..cd8aeb663 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -610,10 +610,11 @@ workflows: requires: - "Build ( darwin / amd64 )" - "Build ( linux / amd64 )" + - "Build ( darwin / arm64 )" filters: branches: - only: - - /^ci\/.*$/ + ignore: + - /^.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -628,6 +629,7 @@ workflows: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - /^ci\/.*$/ [[- range .Networks]] - build-docker: name: "Docker push (lotus-all-in-one / stable / [[.]])" From 4e49b8b671a16112a6bc1346a1588c62c209a9d2 Mon Sep 17 00:00:00 2001 From: Phi Date: Mon, 8 May 2023 13:13:18 +0200 Subject: [PATCH 4/6] Hide lotus-miner legacy markets cmds Hide lotus-miner legacy markets cmds --- cmd/lotus-miner/main.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/cmd/lotus-miner/main.go b/cmd/lotus-miner/main.go index 3cc796168..911e98e26 100644 --- a/cmd/lotus-miner/main.go +++ b/cmd/lotus-miner/main.go @@ -43,16 +43,16 @@ func main() { backupCmd, lcli.WithCategory("chain", actorCmd), lcli.WithCategory("chain", infoCmd), - lcli.WithCategory("market", storageDealsCmd), - lcli.WithCategory("market", retrievalDealsCmd), - lcli.WithCategory("market", dataTransfersCmd), - lcli.WithCategory("market", dagstoreCmd), - lcli.WithCategory("market", indexProvCmd), + lcli.WithCategory("market", setHidden(storageDealsCmd)), + lcli.WithCategory("market", setHidden(retrievalDealsCmd)), + lcli.WithCategory("market", setHidden(dataTransfersCmd)), + lcli.WithCategory("market", setHidden(dagstoreCmd)), + lcli.WithCategory("market", setHidden(indexProvCmd)), lcli.WithCategory("storage", sectorsCmd), lcli.WithCategory("storage", provingCmd), lcli.WithCategory("storage", storageCmd), lcli.WithCategory("storage", sealingCmd), - lcli.WithCategory("retrieval", piecesCmd), + lcli.WithCategory("retrieval", setHidden(piecesCmd)), } jaeger := tracing.SetupJaegerTracing("lotus") @@ -86,6 +86,7 @@ func main() { // adapt the Net* commands to always hit the node running the markets // subsystem, as that is the only one that runs a libp2p node. netCmd := *lcli.NetCmd // make a copy. + netCmd.Hidden = true prev := netCmd.Before netCmd.Before = func(c *cli.Context) error { if prev != nil { @@ -137,11 +138,12 @@ func main() { &cli.StringFlag{ Name: FlagMarketsRepo, EnvVars: []string{"LOTUS_MARKETS_PATH"}, - Usage: fmt.Sprintf("Markets repo path"), + Hidden: true, }, &cli.BoolFlag{ - Name: "call-on-markets", - Usage: "(experimental; may be removed) call this command against a markets node; use only with common commands like net, auth, pprof, etc. whose target may be ambiguous", + Name: "call-on-markets", + Usage: "(experimental; may be removed) call this command against a markets node; use only with common commands like net, auth, pprof, etc. whose target may be ambiguous", + Hidden: true, }, cliutil.FlagVeryVerbose, }, @@ -190,3 +192,8 @@ func getActorAddress(ctx context.Context, cctx *cli.Context) (maddr address.Addr return maddr, nil } + +func setHidden(cmd *cli.Command) *cli.Command { + cmd.Hidden = true + return cmd +} From f03c06964f8381fd6c3356182fc59c1dcfe7338a Mon Sep 17 00:00:00 2001 From: Phi Date: Mon, 8 May 2023 13:42:53 +0200 Subject: [PATCH 5/6] Make docsgen-cli Make docsgen-cli --- documentation/en/cli-lotus-miner.md | 1047 --------------------------- 1 file changed, 1047 deletions(-) diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index cc1943065..c728c684a 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -25,16 +25,6 @@ COMMANDS: log Manage logging wait-api Wait for lotus api to come online fetch-params Fetch proving parameters - MARKET: - storage-deals Manage storage deals and related configuration - retrieval-deals Manage retrieval deals and related configuration - data-transfers Manage data transfers - dagstore Manage the dagstore on the markets subsystem - index Manage the index provider on the markets subsystem - NETWORK: - net Manage P2P Network - RETRIEVAL: - pieces interact with the piecestore STORAGE: sectors interact with sector store proving View proving information @@ -43,10 +33,8 @@ COMMANDS: GLOBAL OPTIONS: --actor value, -a value specify other actor to query / manipulate - --call-on-markets (experimental; may be removed) call this command against a markets node; use only with common commands like net, auth, pprof, etc. whose target may be ambiguous (default: false) --color use color in display output (default: depends on output being a TTY) --help, -h show help (default: false) - --markets-repo value Markets repo path [$LOTUS_MARKETS_PATH] --miner-repo value, --storagerepo value Specify miner repo path. flag(storagerepo) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH] --version, -v print the version (default: false) --vv enables very verbose mode, useful for debugging the CLI (default: false) @@ -607,1041 +595,6 @@ OPTIONS: ``` -## lotus-miner storage-deals -``` -NAME: - lotus-miner storage-deals - Manage storage deals and related configuration - -USAGE: - lotus-miner storage-deals command [command options] [arguments...] - -COMMANDS: - import-data Manually import data for a deal - list List all deals for this miner - selection Configure acceptance criteria for storage deal proposals - set-ask Configure the miner's ask - get-ask Print the miner's ask - set-blocklist Set the miner's list of blocklisted piece CIDs - get-blocklist List the contents of the miner's piece CID blocklist - reset-blocklist Remove all entries from the miner's piece CID blocklist - set-seal-duration Set the expected time, in minutes, that you expect sealing sectors to take. Deals that start before this duration will be rejected. - pending-publish list deals waiting in publish queue - retry-publish retry publishing a deal - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner storage-deals import-data -``` -NAME: - lotus-miner storage-deals import-data - Manually import data for a deal - -USAGE: - lotus-miner storage-deals import-data [command options] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner storage-deals list -``` -NAME: - lotus-miner storage-deals list - List all deals for this miner - -USAGE: - lotus-miner storage-deals list [command options] [arguments...] - -OPTIONS: - --format value output format of data, supported: table, json (default: "table") - --verbose, -v (default: false) - --watch watch deal updates in real-time, rather than a one time list (default: false) - -``` - -### lotus-miner storage-deals selection -``` -NAME: - lotus-miner storage-deals selection - Configure acceptance criteria for storage deal proposals - -USAGE: - lotus-miner storage-deals selection command [command options] [arguments...] - -COMMANDS: - list List storage deal proposal selection criteria - reset Reset storage deal proposal selection criteria to default values - reject Configure criteria which necessitate automatic rejection - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner storage-deals selection list -``` -NAME: - lotus-miner storage-deals selection list - List storage deal proposal selection criteria - -USAGE: - lotus-miner storage-deals selection list [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner storage-deals selection reset -``` -NAME: - lotus-miner storage-deals selection reset - Reset storage deal proposal selection criteria to default values - -USAGE: - lotus-miner storage-deals selection reset [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner storage-deals selection reject -``` -NAME: - lotus-miner storage-deals selection reject - Configure criteria which necessitate automatic rejection - -USAGE: - lotus-miner storage-deals selection reject [command options] [arguments...] - -OPTIONS: - --offline (default: false) - --online (default: false) - --unverified (default: false) - --verified (default: false) - -``` - -### lotus-miner storage-deals set-ask -``` -NAME: - lotus-miner storage-deals set-ask - Configure the miner's ask - -USAGE: - lotus-miner storage-deals set-ask [command options] [arguments...] - -OPTIONS: - --max-piece-size SIZE Set maximum piece size (w/bit-padding, in bytes) in ask to SIZE (default: miner sector size) - --min-piece-size SIZE Set minimum piece size (w/bit-padding, in bytes) in ask to SIZE (default: 256B) - --price PRICE Set the price of the ask for unverified deals (specified as FIL / GiB / Epoch) to PRICE. - --verified-price PRICE Set the price of the ask for verified deals (specified as FIL / GiB / Epoch) to PRICE - -``` - -### lotus-miner storage-deals get-ask -``` -NAME: - lotus-miner storage-deals get-ask - Print the miner's ask - -USAGE: - lotus-miner storage-deals get-ask [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner storage-deals set-blocklist -``` -NAME: - lotus-miner storage-deals set-blocklist - Set the miner's list of blocklisted piece CIDs - -USAGE: - lotus-miner storage-deals set-blocklist [command options] [ (optional, will read from stdin if omitted)] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner storage-deals get-blocklist -``` -NAME: - lotus-miner storage-deals get-blocklist - List the contents of the miner's piece CID blocklist - -USAGE: - lotus-miner storage-deals get-blocklist [command options] [arguments...] - -OPTIONS: - -``` - -### lotus-miner storage-deals reset-blocklist -``` -NAME: - lotus-miner storage-deals reset-blocklist - Remove all entries from the miner's piece CID blocklist - -USAGE: - lotus-miner storage-deals reset-blocklist [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner storage-deals set-seal-duration -``` -NAME: - lotus-miner storage-deals set-seal-duration - Set the expected time, in minutes, that you expect sealing sectors to take. Deals that start before this duration will be rejected. - -USAGE: - lotus-miner storage-deals set-seal-duration [command options] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner storage-deals pending-publish -``` -NAME: - lotus-miner storage-deals pending-publish - list deals waiting in publish queue - -USAGE: - lotus-miner storage-deals pending-publish [command options] [arguments...] - -OPTIONS: - --publish-now send a publish message now (default: false) - -``` - -### lotus-miner storage-deals retry-publish -``` -NAME: - lotus-miner storage-deals retry-publish - retry publishing a deal - -USAGE: - lotus-miner storage-deals retry-publish [command options] - -OPTIONS: - --help, -h show help (default: false) - -``` - -## lotus-miner retrieval-deals -``` -NAME: - lotus-miner retrieval-deals - Manage retrieval deals and related configuration - -USAGE: - lotus-miner retrieval-deals command [command options] [arguments...] - -COMMANDS: - selection Configure acceptance criteria for retrieval deal proposals - set-ask Configure the provider's retrieval ask - get-ask Get the provider's current retrieval ask configured by the provider in the ask-store using the set-ask CLI command - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner retrieval-deals selection -``` -NAME: - lotus-miner retrieval-deals selection - Configure acceptance criteria for retrieval deal proposals - -USAGE: - lotus-miner retrieval-deals selection command [command options] [arguments...] - -COMMANDS: - list List retrieval deal proposal selection criteria - reset Reset retrieval deal proposal selection criteria to default values - reject Configure criteria which necessitate automatic rejection - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner retrieval-deals selection list -``` -NAME: - lotus-miner retrieval-deals selection list - List retrieval deal proposal selection criteria - -USAGE: - lotus-miner retrieval-deals selection list [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner retrieval-deals selection reset -``` -NAME: - lotus-miner retrieval-deals selection reset - Reset retrieval deal proposal selection criteria to default values - -USAGE: - lotus-miner retrieval-deals selection reset [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner retrieval-deals selection reject -``` -NAME: - lotus-miner retrieval-deals selection reject - Configure criteria which necessitate automatic rejection - -USAGE: - lotus-miner retrieval-deals selection reject [command options] [arguments...] - -OPTIONS: - --offline (default: false) - --online (default: false) - -``` - -### lotus-miner retrieval-deals set-ask -``` -NAME: - lotus-miner retrieval-deals set-ask - Configure the provider's retrieval ask - -USAGE: - lotus-miner retrieval-deals set-ask [command options] [arguments...] - -OPTIONS: - --payment-interval value Set the payment interval (in bytes) for retrieval (default: 1MiB) - --payment-interval-increase value Set the payment interval increase (in bytes) for retrieval (default: 1MiB) - --price value Set the price of the ask for retrievals (FIL/GiB) - --unseal-price value Set the price to unseal - -``` - -### lotus-miner retrieval-deals get-ask -``` -NAME: - lotus-miner retrieval-deals get-ask - Get the provider's current retrieval ask configured by the provider in the ask-store using the set-ask CLI command - -USAGE: - lotus-miner retrieval-deals get-ask [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -## lotus-miner data-transfers -``` -NAME: - lotus-miner data-transfers - Manage data transfers - -USAGE: - lotus-miner data-transfers command [command options] [arguments...] - -COMMANDS: - list List ongoing data transfers for this miner - restart Force restart a stalled data transfer - cancel Force cancel a data transfer - diagnostics Get detailed diagnostics on active transfers with a specific peer - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner data-transfers list -``` -NAME: - lotus-miner data-transfers list - List ongoing data transfers for this miner - -USAGE: - lotus-miner data-transfers list [command options] [arguments...] - -OPTIONS: - --completed show completed data transfers (default: false) - --show-failed show failed/cancelled transfers (default: false) - --verbose, -v print verbose transfer details (default: false) - --watch watch deal updates in real-time, rather than a one time list (default: false) - -``` - -### lotus-miner data-transfers restart -``` -NAME: - lotus-miner data-transfers restart - Force restart a stalled data transfer - -USAGE: - lotus-miner data-transfers restart [command options] [arguments...] - -OPTIONS: - --initiator specify only transfers where peer is/is not initiator (default: false) - --peerid value narrow to transfer with specific peer - -``` - -### lotus-miner data-transfers cancel -``` -NAME: - lotus-miner data-transfers cancel - Force cancel a data transfer - -USAGE: - lotus-miner data-transfers cancel [command options] [arguments...] - -OPTIONS: - --cancel-timeout value time to wait for cancel to be sent to client (default: 5s) - --initiator specify only transfers where peer is/is not initiator (default: false) - --peerid value narrow to transfer with specific peer - -``` - -### lotus-miner data-transfers diagnostics -``` -NAME: - lotus-miner data-transfers diagnostics - Get detailed diagnostics on active transfers with a specific peer - -USAGE: - lotus-miner data-transfers diagnostics [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -## lotus-miner dagstore -``` -NAME: - lotus-miner dagstore - Manage the dagstore on the markets subsystem - -USAGE: - lotus-miner dagstore command [command options] [arguments...] - -COMMANDS: - list-shards List all shards known to the dagstore, with their current status - register-shard Register a shard - initialize-shard Initialize the specified shard - recover-shard Attempt to recover a shard in errored state - initialize-all Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default - gc Garbage collect the dagstore - lookup-pieces Lookup pieces that a given CID belongs to - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner dagstore list-shards -``` -NAME: - lotus-miner dagstore list-shards - List all shards known to the dagstore, with their current status - -USAGE: - lotus-miner dagstore list-shards [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner dagstore register-shard -``` -NAME: - lotus-miner dagstore register-shard - Register a shard - -USAGE: - lotus-miner dagstore register-shard [command options] [key] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner dagstore initialize-shard -``` -NAME: - lotus-miner dagstore initialize-shard - Initialize the specified shard - -USAGE: - lotus-miner dagstore initialize-shard [command options] [key] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner dagstore recover-shard -``` -NAME: - lotus-miner dagstore recover-shard - Attempt to recover a shard in errored state - -USAGE: - lotus-miner dagstore recover-shard [command options] [key] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner dagstore initialize-all -``` -NAME: - lotus-miner dagstore initialize-all - Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default - -USAGE: - lotus-miner dagstore initialize-all [command options] [arguments...] - -OPTIONS: - --concurrency value maximum shards to initialize concurrently at a time; use 0 for unlimited (default: 0) - --include-sealed initialize sealed pieces as well (default: false) - -``` - -### lotus-miner dagstore gc -``` -NAME: - lotus-miner dagstore gc - Garbage collect the dagstore - -USAGE: - lotus-miner dagstore gc [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner dagstore lookup-pieces -``` -NAME: - lotus-miner dagstore lookup-pieces - Lookup pieces that a given CID belongs to - -USAGE: - lotus-miner dagstore lookup-pieces [command options] - -OPTIONS: - --help, -h show help (default: false) - -``` - -## lotus-miner index -``` -NAME: - lotus-miner index - Manage the index provider on the markets subsystem - -USAGE: - lotus-miner index command [command options] [arguments...] - -COMMANDS: - announce Announce a deal to indexers so they can download its index - announce-all Announce all active deals to indexers so they can download the indices - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner index announce -``` -NAME: - lotus-miner index announce - Announce a deal to indexers so they can download its index - -USAGE: - lotus-miner index announce [command options] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner index announce-all -``` -NAME: - lotus-miner index announce-all - Announce all active deals to indexers so they can download the indices - -USAGE: - lotus-miner index announce-all [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -## lotus-miner net -``` -NAME: - lotus-miner net - Manage P2P Network - -USAGE: - lotus-miner net command [command options] [arguments...] - -COMMANDS: - peers Print peers - ping Ping peers - connect Connect to a peer - disconnect Disconnect from a peer - listen List listen addresses - id Get node identity - find-peer, findpeer Find the addresses of a given peerID - scores Print peers' pubsub scores - reachability Print information about reachability from the internet - bandwidth Print bandwidth usage information - block Manage network connection gating rules - stat Report resource usage for a scope - limit Get or set resource limits for a scope - protect Add one or more peer IDs to the list of protected peer connections - unprotect Remove one or more peer IDs from the list of protected peer connections. - list-protected List the peer IDs with protected connection. - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner net peers -``` -NAME: - lotus-miner net peers - Print peers - -USAGE: - lotus-miner net peers [command options] [arguments...] - -OPTIONS: - --agent, -a Print agent name (default: false) - --extended, -x Print extended peer information in json (default: false) - -``` - -### lotus-miner net ping -``` -NAME: - lotus-miner net ping - Ping peers - -USAGE: - lotus-miner net ping [command options] [peerMultiaddr] - -OPTIONS: - --count value, -c value specify the number of times it should ping (default: 10) - --interval value, -i value minimum time between pings (default: 1s) - -``` - -### lotus-miner net connect -``` -NAME: - lotus-miner net connect - Connect to a peer - -USAGE: - lotus-miner net connect [command options] [peerMultiaddr|minerActorAddress] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner net disconnect -``` -NAME: - lotus-miner net disconnect - Disconnect from a peer - -USAGE: - lotus-miner net disconnect [command options] [peerID] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner net listen -``` -NAME: - lotus-miner net listen - List listen addresses - -USAGE: - lotus-miner net listen [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner net id -``` -NAME: - lotus-miner net id - Get node identity - -USAGE: - lotus-miner net id [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner net find-peer, findpeer -``` -``` - -### lotus-miner net scores -``` -NAME: - lotus-miner net scores - Print peers' pubsub scores - -USAGE: - lotus-miner net scores [command options] [arguments...] - -OPTIONS: - --extended, -x print extended peer scores in json (default: false) - -``` - -### lotus-miner net reachability -``` -NAME: - lotus-miner net reachability - Print information about reachability from the internet - -USAGE: - lotus-miner net reachability [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner net bandwidth -``` -NAME: - lotus-miner net bandwidth - Print bandwidth usage information - -USAGE: - lotus-miner net bandwidth [command options] [arguments...] - -OPTIONS: - --by-peer list bandwidth usage by peer (default: false) - --by-protocol list bandwidth usage by protocol (default: false) - -``` - -### lotus-miner net block -``` -NAME: - lotus-miner net block - Manage network connection gating rules - -USAGE: - lotus-miner net block command [command options] [arguments...] - -COMMANDS: - add Add connection gating rules - remove Remove connection gating rules - list list connection gating rules - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner net block add -``` -NAME: - lotus-miner net block add - Add connection gating rules - -USAGE: - lotus-miner net block add command [command options] [arguments...] - -COMMANDS: - peer Block a peer - ip Block an IP address - subnet Block an IP subnet - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -##### lotus-miner net block add peer -``` -NAME: - lotus-miner net block add peer - Block a peer - -USAGE: - lotus-miner net block add peer [command options] ... - -OPTIONS: - --help, -h show help (default: false) - -``` - -##### lotus-miner net block add ip -``` -NAME: - lotus-miner net block add ip - Block an IP address - -USAGE: - lotus-miner net block add ip [command options] ... - -OPTIONS: - --help, -h show help (default: false) - -``` - -##### lotus-miner net block add subnet -``` -NAME: - lotus-miner net block add subnet - Block an IP subnet - -USAGE: - lotus-miner net block add subnet [command options] ... - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner net block remove -``` -NAME: - lotus-miner net block remove - Remove connection gating rules - -USAGE: - lotus-miner net block remove command [command options] [arguments...] - -COMMANDS: - peer Unblock a peer - ip Unblock an IP address - subnet Unblock an IP subnet - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -##### lotus-miner net block remove peer -``` -NAME: - lotus-miner net block remove peer - Unblock a peer - -USAGE: - lotus-miner net block remove peer [command options] ... - -OPTIONS: - --help, -h show help (default: false) - -``` - -##### lotus-miner net block remove ip -``` -NAME: - lotus-miner net block remove ip - Unblock an IP address - -USAGE: - lotus-miner net block remove ip [command options] ... - -OPTIONS: - --help, -h show help (default: false) - -``` - -##### lotus-miner net block remove subnet -``` -NAME: - lotus-miner net block remove subnet - Unblock an IP subnet - -USAGE: - lotus-miner net block remove subnet [command options] ... - -OPTIONS: - --help, -h show help (default: false) - -``` - -#### lotus-miner net block list -``` -NAME: - lotus-miner net block list - list connection gating rules - -USAGE: - lotus-miner net block list [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner net stat -``` -NAME: - lotus-miner net stat - Report resource usage for a scope - -USAGE: - lotus-miner net stat [command options] scope - -DESCRIPTION: - Report resource usage for a scope. - - The scope can be one of the following: - - system -- reports the system aggregate resource usage. - - transient -- reports the transient resource usage. - - svc: -- reports the resource usage of a specific service. - - proto: -- reports the resource usage of a specific protocol. - - peer: -- reports the resource usage of a specific peer. - - all -- reports the resource usage for all currently active scopes. - - -OPTIONS: - --json (default: false) - -``` - -### lotus-miner net limit -``` -NAME: - lotus-miner net limit - Get or set resource limits for a scope - -USAGE: - lotus-miner net limit [command options] scope [limit] - -DESCRIPTION: - Get or set resource limits for a scope. - - The scope can be one of the following: - - system -- reports the system aggregate resource usage. - - transient -- reports the transient resource usage. - - svc: -- reports the resource usage of a specific service. - - proto: -- reports the resource usage of a specific protocol. - - peer: -- reports the resource usage of a specific peer. - - The limit is json-formatted, with the same structure as the limits file. - - -OPTIONS: - --set set the limit for a scope (default: false) - -``` - -### lotus-miner net protect -``` -NAME: - lotus-miner net protect - Add one or more peer IDs to the list of protected peer connections - -USAGE: - lotus-miner net protect [command options] [...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner net unprotect -``` -NAME: - lotus-miner net unprotect - Remove one or more peer IDs from the list of protected peer connections. - -USAGE: - lotus-miner net unprotect [command options] [...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner net list-protected -``` -NAME: - lotus-miner net list-protected - List the peer IDs with protected connection. - -USAGE: - lotus-miner net list-protected [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -## lotus-miner pieces -``` -NAME: - lotus-miner pieces - interact with the piecestore - -USAGE: - lotus-miner pieces command [command options] [arguments...] - -DESCRIPTION: - The piecestore is a database that tracks and manages data that is made available to the retrieval market - -COMMANDS: - list-pieces list registered pieces - list-cids list registered payload CIDs - piece-info get registered information for a given piece CID - cid-info get registered information for a given payload CID - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner pieces list-pieces -``` -NAME: - lotus-miner pieces list-pieces - list registered pieces - -USAGE: - lotus-miner pieces list-pieces [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner pieces list-cids -``` -NAME: - lotus-miner pieces list-cids - list registered payload CIDs - -USAGE: - lotus-miner pieces list-cids [command options] [arguments...] - -OPTIONS: - --verbose, -v (default: false) - -``` - -### lotus-miner pieces piece-info -``` -NAME: - lotus-miner pieces piece-info - get registered information for a given piece CID - -USAGE: - lotus-miner pieces piece-info [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - -### lotus-miner pieces cid-info -``` -NAME: - lotus-miner pieces cid-info - get registered information for a given payload CID - -USAGE: - lotus-miner pieces cid-info [command options] [arguments...] - -OPTIONS: - --help, -h show help (default: false) - -``` - ## lotus-miner sectors ``` NAME: From 60d576241c7efa28f83c67cf5b96d6753ed5ea3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 10 May 2023 21:43:42 +0200 Subject: [PATCH 6/6] fix: sched: Address GET_32G_MAX_CONCURRENT regression (#10850) * Fix 1.21 regression: GET_32G_MAX_CONCURRENT + mixed prepared/executing leads to stuck scheduler If you have 12 GET tasks and GET_32G_MAX_CONCURRENT=1, sealing jobs will only show assigned tasks for GET of the miner and is stuck. I believe this to be a regression of 1.21 unifying the counters, in the case of GETs where PrepType and TaskType both being seal/v0/fetch leading to a state where tasks are blocked since already counted towards the limit. * itests: Repro issue from PR #10633 * make counters int (non-working) * fix: worker sched: Send taskDone notifs after tasks are done * itests: Make TestPledgeMaxConcurrentGet actually reproduce the issue * make the linter happy --------- Co-authored-by: Steffen Butzer --- .circleci/config.yml | 6 ++ itests/sealing_resources_test.go | 64 +++++++++++++++++++ storage/sealer/sched_assigner_common.go | 2 +- storage/sealer/sched_assigner_darts.go | 4 +- storage/sealer/sched_assigner_spread.go | 4 +- storage/sealer/sched_assigner_spread_tasks.go | 4 +- storage/sealer/sched_assigner_utilization.go | 4 +- storage/sealer/sched_post.go | 5 +- storage/sealer/sched_resources.go | 63 +++++++++++------- storage/sealer/sched_test.go | 4 +- storage/sealer/sched_worker.go | 29 +++++---- 11 files changed, 142 insertions(+), 47 deletions(-) create mode 100644 itests/sealing_resources_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 9038fdb8f..5fcb83145 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -882,6 +882,12 @@ workflows: - build suite: itest-sdr_upgrade target: "./itests/sdr_upgrade_test.go" + - test: + name: test-itest-sealing_resources + requires: + - build + suite: itest-sealing_resources + target: "./itests/sealing_resources_test.go" - test: name: test-itest-sector_finalize_early requires: diff --git a/itests/sealing_resources_test.go b/itests/sealing_resources_test.go new file mode 100644 index 000000000..85779fd88 --- /dev/null +++ b/itests/sealing_resources_test.go @@ -0,0 +1,64 @@ +package itests + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/lotus/itests/kit" + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +// Regression check for a fix introduced in https://github.com/filecoin-project/lotus/pull/10633 +func TestPledgeMaxConcurrentGet(t *testing.T) { + require.NoError(t, os.Setenv("GET_2K_MAX_CONCURRENT", "1")) + t.Cleanup(func() { + require.NoError(t, os.Unsetenv("GET_2K_MAX_CONCURRENT")) + }) + + kit.QuietMiningLogs() + + blockTime := 50 * time.Millisecond + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, miner, ens := kit.EnsembleMinimal(t, kit.NoStorage()) // no mock proofs + ens.InterconnectAll().BeginMiningMustPost(blockTime) + + // separate sealed and storage paths so that finalize move needs to happen + miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) { + meta.CanSeal = true + }) + miner.AddStorage(ctx, t, func(meta *storiface.LocalStorageMeta) { + meta.CanStore = true + }) + + // NOTE: This test only repros the issue when Fetch tasks take ~10s, there's + // no great way to do that in a non-horribly-hacky way + + /* The horribly hacky way: + + diff --git a/storage/sealer/sched_worker.go b/storage/sealer/sched_worker.go + index 35acd755d..76faec859 100644 + --- a/storage/sealer/sched_worker.go + +++ b/storage/sealer/sched_worker.go + @@ -513,6 +513,10 @@ func (sw *schedWorker) startProcessingTask(req *WorkerRequest) error { + tw.start() + err = <-werr + + + if req.TaskType == sealtasks.TTFetch { + + time.Sleep(10 * time.Second) + + } + + + select { + case req.ret <- workerResponse{err: err}: + case <-req.Ctx.Done(): + + */ + + miner.PledgeSectors(ctx, 3, 0, nil) +} diff --git a/storage/sealer/sched_assigner_common.go b/storage/sealer/sched_assigner_common.go index d676d410d..ffc21b0dd 100644 --- a/storage/sealer/sched_assigner_common.go +++ b/storage/sealer/sched_assigner_common.go @@ -103,7 +103,7 @@ func (a *AssignerCommon) TrySched(sh *Scheduler) { needRes := worker.Info.Resources.ResourceSpec(task.Sector.ProofType, task.TaskType) // TODO: allow bigger windows - if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), needRes, windowRequest.Worker, "schedAcceptable", worker.Info) { + if !windows[wnd].Allocated.CanHandleRequest(task.SchedId, task.SealTask(), needRes, windowRequest.Worker, "schedAcceptable", worker.Info) { continue } diff --git a/storage/sealer/sched_assigner_darts.go b/storage/sealer/sched_assigner_darts.go index e28b70e78..134698fbf 100644 --- a/storage/sealer/sched_assigner_darts.go +++ b/storage/sealer/sched_assigner_darts.go @@ -37,7 +37,7 @@ func RandomWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows [] log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) - if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + if !windows[wnd].Allocated.CanHandleRequest(task.SchedId, task.SealTask(), res, wid, "schedAssign", w.Info) { continue } @@ -71,7 +71,7 @@ func RandomWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, windows [] "worker", bestWid, "choices", len(choices)) - windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + windows[selectedWindow].Allocated.Add(task.SchedId, task.SealTask(), info.Resources, needRes) windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) rmQueue = append(rmQueue, sqi) diff --git a/storage/sealer/sched_assigner_spread.go b/storage/sealer/sched_assigner_spread.go index 0a62b7406..b1ac4c8e9 100644 --- a/storage/sealer/sched_assigner_spread.go +++ b/storage/sealer/sched_assigner_spread.go @@ -35,7 +35,7 @@ func SpreadWS(queued bool) func(sh *Scheduler, queueLen int, acceptableWindows [ log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) - if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + if !windows[wnd].Allocated.CanHandleRequest(task.SchedId, task.SealTask(), res, wid, "schedAssign", w.Info) { continue } @@ -71,7 +71,7 @@ func SpreadWS(queued bool) func(sh *Scheduler, queueLen int, acceptableWindows [ "assigned", bestAssigned) workerAssigned[bestWid]++ - windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + windows[selectedWindow].Allocated.Add(task.SchedId, task.SealTask(), info.Resources, needRes) windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) rmQueue = append(rmQueue, sqi) diff --git a/storage/sealer/sched_assigner_spread_tasks.go b/storage/sealer/sched_assigner_spread_tasks.go index 09cf98046..f98e7b745 100644 --- a/storage/sealer/sched_assigner_spread_tasks.go +++ b/storage/sealer/sched_assigner_spread_tasks.go @@ -41,7 +41,7 @@ func SpreadTasksWS(queued bool) func(sh *Scheduler, queueLen int, acceptableWind log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) - if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + if !windows[wnd].Allocated.CanHandleRequest(task.SchedId, task.SealTask(), res, wid, "schedAssign", w.Info) { continue } @@ -80,7 +80,7 @@ func SpreadTasksWS(queued bool) func(sh *Scheduler, queueLen int, acceptableWind "assigned", bestAssigned) workerAssigned[bestWid]++ - windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + windows[selectedWindow].Allocated.Add(task.SchedId, task.SealTask(), info.Resources, needRes) windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) rmQueue = append(rmQueue, sqi) diff --git a/storage/sealer/sched_assigner_utilization.go b/storage/sealer/sched_assigner_utilization.go index 1e75d904a..c81c9f187 100644 --- a/storage/sealer/sched_assigner_utilization.go +++ b/storage/sealer/sched_assigner_utilization.go @@ -35,7 +35,7 @@ func LowestUtilizationWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, log.Debugf("SCHED try assign sqi:%d sector %d to window %d (awi:%d)", sqi, task.Sector.ID.Number, wnd, i) // TODO: allow bigger windows - if !windows[wnd].Allocated.CanHandleRequest(task.SealTask(), res, wid, "schedAssign", w.Info) { + if !windows[wnd].Allocated.CanHandleRequest(task.SchedId, task.SealTask(), res, wid, "schedAssign", w.Info) { continue } @@ -82,7 +82,7 @@ func LowestUtilizationWS(sh *Scheduler, queueLen int, acceptableWindows [][]int, "worker", bestWid, "utilization", bestUtilization) - workerUtil[bestWid] += windows[selectedWindow].Allocated.Add(task.SealTask(), info.Resources, needRes) + workerUtil[bestWid] += windows[selectedWindow].Allocated.Add(task.SchedId, task.SealTask(), info.Resources, needRes) windows[selectedWindow].Todo = append(windows[selectedWindow].Todo, task) rmQueue = append(rmQueue, sqi) diff --git a/storage/sealer/sched_post.go b/storage/sealer/sched_post.go index 0e0c39768..c6bd81829 100644 --- a/storage/sealer/sched_post.go +++ b/storage/sealer/sched_post.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/google/uuid" "github.com/hashicorp/go-multierror" "golang.org/x/xerrors" @@ -110,7 +111,7 @@ func (ps *poStScheduler) Schedule(ctx context.Context, primary bool, spt abi.Reg for i, selected := range candidates { worker := ps.workers[selected.id] - err := worker.active.withResources(selected.id, worker.Info, ps.postType.SealTask(spt), selected.res, &ps.lk, func() error { + err := worker.active.withResources(uuid.UUID{}, selected.id, worker.Info, ps.postType.SealTask(spt), selected.res, &ps.lk, func() error { ps.lk.Unlock() defer ps.lk.Lock() @@ -148,7 +149,7 @@ func (ps *poStScheduler) readyWorkers(spt abi.RegisteredSealProof) (bool, []cand continue } - if !wr.active.CanHandleRequest(ps.postType.SealTask(spt), needRes, wid, "post-readyWorkers", wr.Info) { + if !wr.active.CanHandleRequest(uuid.UUID{}, ps.postType.SealTask(spt), needRes, wid, "post-readyWorkers", wr.Info) { continue } diff --git a/storage/sealer/sched_resources.go b/storage/sealer/sched_resources.go index 597f36dbe..a423def9f 100644 --- a/storage/sealer/sched_resources.go +++ b/storage/sealer/sched_resources.go @@ -3,6 +3,8 @@ package sealer import ( "sync" + "github.com/google/uuid" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" "github.com/filecoin-project/lotus/storage/sealer/storiface" ) @@ -20,7 +22,7 @@ type ActiveResources struct { } type taskCounter struct { - taskCounters map[sealtasks.SealTaskType]int + taskCounters map[sealtasks.SealTaskType]map[uuid.UUID]int // this lock is technically redundant, as ActiveResources is always accessed // with the worker lock, but let's not panic if we ever change that @@ -29,34 +31,48 @@ type taskCounter struct { func newTaskCounter() *taskCounter { return &taskCounter{ - taskCounters: map[sealtasks.SealTaskType]int{}, + taskCounters: make(map[sealtasks.SealTaskType]map[uuid.UUID]int), } } -func (tc *taskCounter) Add(tt sealtasks.SealTaskType) { +func (tc *taskCounter) Add(tt sealtasks.SealTaskType, schedID uuid.UUID) { tc.lk.Lock() defer tc.lk.Unlock() - tc.taskCounters[tt]++ + tc.getUnlocked(tt)[schedID]++ } -func (tc *taskCounter) Free(tt sealtasks.SealTaskType) { +func (tc *taskCounter) Free(tt sealtasks.SealTaskType, schedID uuid.UUID) { tc.lk.Lock() defer tc.lk.Unlock() - tc.taskCounters[tt]-- + m := tc.getUnlocked(tt) + if m[schedID] <= 1 { + delete(m, schedID) + } else { + m[schedID]-- + } } -func (tc *taskCounter) Get(tt sealtasks.SealTaskType) int { - tc.lk.Lock() - defer tc.lk.Unlock() +func (tc *taskCounter) getUnlocked(tt sealtasks.SealTaskType) map[uuid.UUID]int { + if tc.taskCounters[tt] == nil { + tc.taskCounters[tt] = make(map[uuid.UUID]int) + } + return tc.taskCounters[tt] } +func (tc *taskCounter) Get(tt sealtasks.SealTaskType) map[uuid.UUID]int { + tc.lk.Lock() + defer tc.lk.Unlock() + + return tc.getUnlocked(tt) +} + func (tc *taskCounter) Sum() int { tc.lk.Lock() defer tc.lk.Unlock() sum := 0 for _, v := range tc.taskCounters { - sum += v + sum += len(v) } return sum } @@ -64,8 +80,8 @@ func (tc *taskCounter) Sum() int { func (tc *taskCounter) ForEach(cb func(tt sealtasks.SealTaskType, count int)) { tc.lk.Lock() defer tc.lk.Unlock() - for tt, count := range tc.taskCounters { - cb(tt, count) + for tt, v := range tc.taskCounters { + cb(tt, len(v)) } } @@ -75,8 +91,8 @@ func NewActiveResources(tc *taskCounter) *ActiveResources { } } -func (a *ActiveResources) withResources(id storiface.WorkerID, wr storiface.WorkerInfo, tt sealtasks.SealTaskType, r storiface.Resources, locker sync.Locker, cb func() error) error { - for !a.CanHandleRequest(tt, r, id, "withResources", wr) { +func (a *ActiveResources) withResources(schedID uuid.UUID, id storiface.WorkerID, wr storiface.WorkerInfo, tt sealtasks.SealTaskType, r storiface.Resources, locker sync.Locker, cb func() error) error { + for !a.CanHandleRequest(schedID, tt, r, id, "withResources", wr) { if a.cond == nil { a.cond = sync.NewCond(locker) } @@ -85,11 +101,11 @@ func (a *ActiveResources) withResources(id storiface.WorkerID, wr storiface.Work a.waiting-- } - a.Add(tt, wr.Resources, r) + a.Add(schedID, tt, wr.Resources, r) err := cb() - a.Free(tt, wr.Resources, r) + a.Free(schedID, tt, wr.Resources, r) return err } @@ -100,7 +116,7 @@ func (a *ActiveResources) hasWorkWaiting() bool { } // add task resources to ActiveResources and return utilization difference -func (a *ActiveResources) Add(tt sealtasks.SealTaskType, wr storiface.WorkerResources, r storiface.Resources) float64 { +func (a *ActiveResources) Add(schedID uuid.UUID, tt sealtasks.SealTaskType, wr storiface.WorkerResources, r storiface.Resources) float64 { startUtil := a.utilization(wr) if r.GPUUtilization > 0 { @@ -109,19 +125,19 @@ func (a *ActiveResources) Add(tt sealtasks.SealTaskType, wr storiface.WorkerReso a.cpuUse += r.Threads(wr.CPUs, len(wr.GPUs)) a.memUsedMin += r.MinMemory a.memUsedMax += r.MaxMemory - a.taskCounters.Add(tt) + a.taskCounters.Add(tt, schedID) return a.utilization(wr) - startUtil } -func (a *ActiveResources) Free(tt sealtasks.SealTaskType, wr storiface.WorkerResources, r storiface.Resources) { +func (a *ActiveResources) Free(schedID uuid.UUID, tt sealtasks.SealTaskType, wr storiface.WorkerResources, r storiface.Resources) { if r.GPUUtilization > 0 { a.gpuUsed -= r.GPUUtilization } a.cpuUse -= r.Threads(wr.CPUs, len(wr.GPUs)) a.memUsedMin -= r.MinMemory a.memUsedMax -= r.MaxMemory - a.taskCounters.Free(tt) + a.taskCounters.Free(tt, schedID) if a.cond != nil { a.cond.Broadcast() @@ -130,9 +146,10 @@ func (a *ActiveResources) Free(tt sealtasks.SealTaskType, wr storiface.WorkerRes // CanHandleRequest evaluates if the worker has enough available resources to // handle the request. -func (a *ActiveResources) CanHandleRequest(tt sealtasks.SealTaskType, needRes storiface.Resources, wid storiface.WorkerID, caller string, info storiface.WorkerInfo) bool { +func (a *ActiveResources) CanHandleRequest(schedID uuid.UUID, tt sealtasks.SealTaskType, needRes storiface.Resources, wid storiface.WorkerID, caller string, info storiface.WorkerInfo) bool { if needRes.MaxConcurrent > 0 { - if a.taskCounters.Get(tt) >= needRes.MaxConcurrent { + tasks := a.taskCounters.Get(tt) + if len(tasks) >= needRes.MaxConcurrent && (schedID == uuid.UUID{} || tasks[schedID] == 0) { log.Debugf("sched: not scheduling on worker %s for %s; at task limit tt=%s, curcount=%d", wid, caller, tt, a.taskCounters.Get(tt)) return false } @@ -226,7 +243,7 @@ func (a *ActiveResources) taskCount(tt *sealtasks.SealTaskType) int { return a.taskCounters.Sum() } - return a.taskCounters.Get(*tt) + return len(a.taskCounters.Get(*tt)) } func (wh *WorkerHandle) Utilization() float64 { diff --git a/storage/sealer/sched_test.go b/storage/sealer/sched_test.go index 07731e934..2e2b05ab2 100644 --- a/storage/sealer/sched_test.go +++ b/storage/sealer/sched_test.go @@ -698,7 +698,7 @@ func TestWindowCompact(t *testing.T) { TaskType: task, Sector: storiface.SectorRef{ProofType: spt}, }) - window.Allocated.Add(task.SealTask(spt), wh.Info.Resources, storiface.ResourceTable[task][spt]) + window.Allocated.Add(uuid.UUID{}, task.SealTask(spt), wh.Info.Resources, storiface.ResourceTable[task][spt]) } wh.activeWindows = append(wh.activeWindows, window) @@ -717,7 +717,7 @@ func TestWindowCompact(t *testing.T) { for ti, task := range tasks { require.Equal(t, task, wh.activeWindows[wi].Todo[ti].TaskType, "%d, %d", wi, ti) - expectRes.Add(task.SealTask(spt), wh.Info.Resources, storiface.ResourceTable[task][spt]) + expectRes.Add(uuid.UUID{}, task.SealTask(spt), wh.Info.Resources, storiface.ResourceTable[task][spt]) } require.Equal(t, expectRes.cpuUse, wh.activeWindows[wi].Allocated.cpuUse, "%d", wi) diff --git a/storage/sealer/sched_worker.go b/storage/sealer/sched_worker.go index b6efc851a..35acd755d 100644 --- a/storage/sealer/sched_worker.go +++ b/storage/sealer/sched_worker.go @@ -294,14 +294,14 @@ func (sw *schedWorker) workerCompactWindows() { for ti, todo := range window.Todo { needRes := worker.Info.Resources.ResourceSpec(todo.Sector.ProofType, todo.TaskType) - if !lower.Allocated.CanHandleRequest(todo.SealTask(), needRes, sw.wid, "compactWindows", worker.Info) { + if !lower.Allocated.CanHandleRequest(todo.SchedId, todo.SealTask(), needRes, sw.wid, "compactWindows", worker.Info) { continue } moved = append(moved, ti) lower.Todo = append(lower.Todo, todo) - lower.Allocated.Add(todo.SealTask(), worker.Info.Resources, needRes) - window.Allocated.Free(todo.SealTask(), worker.Info.Resources, needRes) + lower.Allocated.Add(todo.SchedId, todo.SealTask(), worker.Info.Resources, needRes) + window.Allocated.Free(todo.SchedId, todo.SealTask(), worker.Info.Resources, needRes) } if len(moved) > 0 { @@ -355,7 +355,7 @@ assignLoop: worker.lk.Lock() for t, todo := range firstWindow.Todo { needResPrep := worker.Info.Resources.PrepResourceSpec(todo.Sector.ProofType, todo.TaskType, todo.prepare.PrepType) - if worker.preparing.CanHandleRequest(todo.PrepSealTask(), needResPrep, sw.wid, "startPreparing", worker.Info) { + if worker.preparing.CanHandleRequest(todo.SchedId, todo.PrepSealTask(), needResPrep, sw.wid, "startPreparing", worker.Info) { tidx = t break } @@ -416,7 +416,7 @@ assignLoop: } needRes := worker.Info.Resources.ResourceSpec(todo.Sector.ProofType, todo.TaskType) - if worker.active.CanHandleRequest(todo.SealTask(), needRes, sw.wid, "startPreparing", worker.Info) { + if worker.active.CanHandleRequest(todo.SchedId, todo.SealTask(), needRes, sw.wid, "startPreparing", worker.Info) { tidx = t break } @@ -457,7 +457,7 @@ func (sw *schedWorker) startProcessingTask(req *WorkerRequest) error { needResPrep := w.Info.Resources.PrepResourceSpec(req.Sector.ProofType, req.TaskType, req.prepare.PrepType) w.lk.Lock() - w.preparing.Add(req.PrepSealTask(), w.Info.Resources, needResPrep) + w.preparing.Add(req.SchedId, req.PrepSealTask(), w.Info.Resources, needResPrep) w.lk.Unlock() go func() { @@ -468,7 +468,7 @@ func (sw *schedWorker) startProcessingTask(req *WorkerRequest) error { w.lk.Lock() if err != nil { - w.preparing.Free(req.PrepSealTask(), w.Info.Resources, needResPrep) + w.preparing.Free(req.SchedId, req.PrepSealTask(), w.Info.Resources, needResPrep) w.lk.Unlock() select { @@ -497,11 +497,12 @@ func (sw *schedWorker) startProcessingTask(req *WorkerRequest) error { }() // wait (if needed) for resources in the 'active' window - err = w.active.withResources(sw.wid, w.Info, req.SealTask(), needRes, &w.lk, func() error { - w.preparing.Free(req.PrepSealTask(), w.Info.Resources, needResPrep) + err = w.active.withResources(req.SchedId, sw.wid, w.Info, req.SealTask(), needRes, &w.lk, func() error { + w.preparing.Free(req.SchedId, req.PrepSealTask(), w.Info.Resources, needResPrep) w.lk.Unlock() defer w.lk.Lock() // we MUST return locked from this function + // make sure the worker loop sees that the prepare task has finished select { case sw.taskDone <- struct{}{}: case <-sh.closing: @@ -525,6 +526,12 @@ func (sw *schedWorker) startProcessingTask(req *WorkerRequest) error { w.lk.Unlock() + // make sure the worker loop sees that the task has finished + select { + case sw.taskDone <- struct{}{}: + default: // there is a notification pending already + } + // This error should always be nil, since nothing is setting it, but just to be safe: if err != nil { log.Errorf("error executing worker (withResources): %+v", err) @@ -539,7 +546,7 @@ func (sw *schedWorker) startProcessingReadyTask(req *WorkerRequest) error { needRes := w.Info.Resources.ResourceSpec(req.Sector.ProofType, req.TaskType) - w.active.Add(req.SealTask(), w.Info.Resources, needRes) + w.active.Add(req.SchedId, req.SealTask(), w.Info.Resources, needRes) go func() { // Do the work! @@ -557,7 +564,7 @@ func (sw *schedWorker) startProcessingReadyTask(req *WorkerRequest) error { w.lk.Lock() - w.active.Free(req.SealTask(), w.Info.Resources, needRes) + w.active.Free(req.SchedId, req.SealTask(), w.Info.Resources, needRes) select { case sw.taskDone <- struct{}{}: