Merge pull request #7908 from filecoin-project/release/v1.13.2

build: release: v1.13.2
Merge pull request #7907 from filecoin-project/build/v1.13.2prep
2022-01-09 23:12:14 -05:00 · 2022-01-09 23:00:59 -05:00 · 2022-01-09 22:47:41 -05:00 · 2022-01-09 22:39:09 -05:00 · 2022-01-09 22:32:59 -05:00 · 2022-01-09 21:20:05 -05:00
180 changed files with 9342 additions and 4213 deletions
@@ -318,7 +318,7 @@ jobs:
          command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
      - run:
          name: "prepare testground home dir and link test plans"
-          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
+          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup
      - run:
          name: "go get lotus@master"
          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
@@ -328,10 +328,6 @@ jobs:
      - run:
          name: "trigger payment channel stress testplan on taas"
          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger graphsync testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-

  build-macos:
    description: build darwin lotus binary
@@ -809,6 +805,11 @@ workflows:
          suite: itest-deals_padding
          target: "./itests/deals_padding_test.go"
      
+      - test:
+          name: test-itest-deals_partial_retrieval_dm-level
+          suite: itest-deals_partial_retrieval_dm-level
+          target: "./itests/deals_partial_retrieval_dm-level_test.go"
+      
      - test:
          name: test-itest-deals_partial_retrieval
          suite: itest-deals_partial_retrieval
@@ -829,6 +830,11 @@ workflows:
          suite: itest-deals_publish
          target: "./itests/deals_publish_test.go"
      
+      - test:
+          name: test-itest-deals_retry_deal_no_funds
+          suite: itest-deals_retry_deal_no_funds
+          target: "./itests/deals_retry_deal_no_funds_test.go"
+      
      - test:
          name: test-itest-deals
          suite: itest-deals
@@ -939,7 +945,7 @@ workflows:
          codecov-upload: false
          suite: conformance-bleeding-edge
          target: "./conformance"
-          vectors-branch: master
+          vectors-branch: specs-actors-v7
      - trigger-testplans:
          filters:
            branches:
@@ -970,19 +976,10 @@ workflows:
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-appimage:
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - publish:
          requires:
            - build-all
            - build-macos
-            - build-appimage
          filters:
            branches:
              ignore:
@@ -318,7 +318,7 @@ jobs:
          command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
      - run:
          name: "prepare testground home dir and link test plans"
-          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync
+          command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup
      - run:
          name: "go get lotus@master"
          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
@@ -328,10 +328,6 @@ jobs:
      - run:
          name: "trigger payment channel stress testplan on taas"
          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-      - run:
-          name: "trigger graphsync testplan on taas"
-          command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-

  build-macos:
    description: build darwin lotus binary
@@ -789,7 +785,7 @@ workflows:
          codecov-upload: false
          suite: conformance-bleeding-edge
          target: "./conformance"
-          vectors-branch: master
+          vectors-branch: specs-actors-v7
      - trigger-testplans:
          filters:
            branches:
@@ -820,19 +816,10 @@ workflows:
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-appimage:
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - publish:
          requires:
            - build-all
            - build-macos
-            - build-appimage
          filters:
            branches:
              ignore:
@@ -1,5 +1,190 @@
 # Lotus changelog

+# v1.13.2 /  2022-01-09
+
+Lotus v1.13.2 is a *highly recommended* feature release with remarkable retrieval improvements, new features like 
+worker management, schedule enhancements and so on. 
+
+## Highlights
+- 🚀🚀🚀Improve retrieval deal experience
+  - Testing result with MinerX.3 shows the retrieval deal success rate has increased dramatically with faster transfer 
+    speed, you can join or follow along furthur performance testings [here](https://github.com/filecoin-project/lotus/discussions/7874). We recommend application developers to integrate with the new 
+    retrieval APIs to provide a better client experience.
+  - 🌟🌟🌟 Reduce retrieval Time-To-First-Byte over 100x ([#7693](https://github.com/filecoin-project/lotus/pull/7693))
+    - This change makes most free, small retrievals sub-second
+  - 🌟🌟🌟 Partial retrieval ux improvements ([#7610](https://github.com/filecoin-project/lotus/pull/7610))
+    - New retrieval commands for clients:
+      - `lotus client ls`: retrieve and list desired object links
+      - `lotus client cat`: retrieve and print the data from the network
+    - 🌟🌟 The monolith `ClientRetrieve` method was broken into:
+      - `ClientRetrieve` which retrieves data into the local repo (or into an IPFS node if ipfs integration is enabled)
+      - `ClientRetrieveWait` which will wait for the retrieval to complete
+      - `ClientExport` which will export data from the local node
+      - Note: this change only applies to v1 API. v0 API remains unchanged.
+    - 🌟 Support for full ipld selectors was added (for example making it possible to only retrieve list of directories in a deal, without fetching any file data)
+      - To learn more, see [here](https://github.com/filecoin-project/lotus/blob/0523c946f984b22b3f5de8cc3003cc791389527e/api/types.go#L230-L264)
+- 🚀🚀 Sealing scheduler enhancements ([#7703](https://github.com/filecoin-project/lotus/pull/7703),
+  [#7269](https://github.com/filecoin-project/lotus/pull/7269)), [#7714](https://github.com/filecoin-project/lotus/pull/7714)
+  - Workers are now aware of cgroup memory limits
+  - Multiple tasks which use a GPU can be scheduled on a single worker
+  - Workers can override default resource table through env vars
+    - Default value list: https://gist.github.com/magik6k/c0e1c7cd73c1241a9acabc30bf469a43
+- 🚀🚀 Sector storage groups ([#7453](https://github.com/filecoin-project/lotus/pull/7453))
+  - Storage groups allow for better control of data flow between workers, for example, it makes it possible to define that data from PC1 on a given worker has to have it's PC2 step executed on the same worker
+  - To set it up, follow the instructions under the `Sector Storage Group` section [here](https://lotus.filecoin.io/docs/storage-providers/seal-workers/#lotus-worker-co-location)
+
+## New Features
+- Add RLE dump code ([#7691](https://github.com/filecoin-project/lotus/pull/7691))
+- Shed: Add a util to list miner faults ([#7605](https://github.com/filecoin-project/lotus/pull/7605))
+- lotus-shed msg: Decode submessages/msig proposals ([#7639](https://github.com/filecoin-project/lotus/pull/7639))
+- CLI: Add a lotus multisig cancel command ([#7645](https://github.com/filecoin-project/lotus/pull/7645))
+- shed: simple wallet balancer util ([#7414](https://github.com/filecoin-project/lotus/pull/7414))
+  - balancing token balance between multiple accounts
+
+## Improvements
+- Add verbose mode to `lotus-miner pieces list-cids` ([#7699](https://github.com/filecoin-project/lotus/pull/7699))
+- retrieval: Only output matching nodes, MatchPath dagspec ([#7706](https://github.com/filecoin-project/lotus/pull/7706))
+- Cleanup partial retrieval codepaths ( zero functional changes ) ([#7688](https://github.com/filecoin-project/lotus/pull/7688))
+- storage: Use 1M buffers for Tar transfers ([#7681](https://github.com/filecoin-project/lotus/pull/7681))
+- Chore/dm level tests plus merkle proof cars ([#7673](https://github.com/filecoin-project/lotus/pull/7673))
+- Shed: Add a util to create miners more easily ([#7595](https://github.com/filecoin-project/lotus/pull/7595))
+- add timeout flag to wait-api command ([#7592](https://github.com/filecoin-project/lotus/pull/7592))
+- add log for restart windows post scheduler ([#7613](https://github.com/filecoin-project/lotus/pull/7613))
+- remove jaeger envvars ([#7631](https://github.com/filecoin-project/lotus/pull/7631))
+- remove api and jaeger env from docker file ([#7624](https://github.com/filecoin-project/lotus/pull/7624))
+- Wdpost worker: Reduce challenge confidence to 1 epoch ([#7572](https://github.com/filecoin-project/lotus/pull/7572))
+- add additional methods to lotus gateway ([#7644](https://github.com/filecoin-project/lotus/pull/7644))
+- Add caches to lotus-stats and splitcode ([#7329](https://github.com/filecoin-project/lotus/pull/7329))
+- remote store: Remove debug printf ([#7664](https://github.com/filecoin-project/lotus/pull/7664))
+- docsgen-cli: Handle commands with no description correctly ([#7659](https://github.com/filecoin-project/lotus/pull/7659))
+
+## Bug Fixes
+- fix docker logic error ([#7709](https://github.com/filecoin-project/lotus/pull/7709))
+- add missing NodeType tag ([#7559](https://github.com/filecoin-project/lotus/pull/7559))
+- checkCommit should return SectorCommitFailed ([#7555](https://github.com/filecoin-project/lotus/pull/7555))
+- ffiwrapper: Validate PC2 by calling C1 with random seeds ([#7710](https://github.com/filecoin-project/lotus/pull/7710))
+
+## Dependency Updates
+- Update go-graphsync v0.10.6 ([#7708](https://github.com/filecoin-project/lotus/pull/7708))
+- update go-libp2p-pubsub to v0.5.6 ([#7581](https://github.com/filecoin-project/lotus/pull/7581))
+- Update go-state-types ([#7591](https://github.com/filecoin-project/lotus/pull/7591))
+- disable mplex stream muxer ([#7689](https://github.com/filecoin-project/lotus/pull/7689))
+- Bump ws from 5.2.2 to 5.2.3 in /lotuspond/front ([#7660](https://github.com/filecoin-project/lotus/pull/7660))
+- Bump color-string from 1.5.3 to 1.6.0 in /lotuspond/front ([#7658](https://github.com/filecoin-project/lotus/pull/7658))
+- Bump postcss from 7.0.17 to 7.0.39 in /lotuspond/front ([#7657](https://github.com/filecoin-project/lotus/pull/7657))
+- Bump path-parse from 1.0.6 to 1.0.7 in /lotuspond/front ([#7656](https://github.com/filecoin-project/lotus/pull/7656))
+- Bump tmpl from 1.0.4 to 1.0.5 in /lotuspond/front ([#7655](https://github.com/filecoin-project/lotus/pull/7655))
+- Bump url-parse from 1.4.7 to 1.5.3 in /lotuspond/front ([#7654](https://github.com/filecoin-project/lotus/pull/7654))
+- github.com/filecoin-project/go-state-types (v0.1.1-0.20210915140513-d354ccf10379 -> v0.1.1):
+
+## Others
+- Update archive script ([#7690](https://github.com/filecoin-project/lotus/pull/7690))
+
+## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| @magik6k | 89 | +5200/-1818 | 232 |
+| Travis Person | 5 | +1473/-953 | 38 |
+| @arajasek | 6 | +550/-38 | 19 |
+| @clinta | 4 | +393/-123 | 26 |
+| @ribasushi | 3 | +334/-68 | 7 |
+| @jennijuju| 13 | +197/-120 | 67 |
+| @Kubuxu | 10 | +153/-30 | 10 |
+| @coryschwartz | 6 | +18/-26 | 6 |
+| Marten Seemann | 2 | +6/-34 | 5 |
+| @vyzo | 1 | +3/-3 | 2 |
+| @hannahhoward | 1 | +3/-3 | 2 |
+| @zenground0 | 2 | +2/-2 | 2 |
+| @yaohcn | 2 | +2/-2 | 2 |
+| @jennijuju | 1 | +1/-1 | 1 |
+| @hunjixin | 1 | +1/-0 | 1 |
+
+    
+
+# v1.13.1 / 2021-11-26
+
+This is an optional Lotus v1.13.1 release.
+
+## New Features
+- Shed: Add a util to find miner based on peerid ([filecoin-project/lotus#7544](https://github.com/filecoin-project/lotus/pull/7544))
+- Collect and expose graphsync metrics  ([filecoin-project/lotus#7542](https://github.com/filecoin-project/lotus/pull/7542))
+- Shed: Add a util to find the most recent null tipset ([filecoin-project/lotus#7456](https://github.com/filecoin-project/lotus/pull/7456))
+
+## Improvements
+- Show prepared tasks in sealing jobs ([filecoin-project/lotus#7527](https://github.com/filecoin-project/lotus/pull/7527))
+- To make Deep happy ([filecoin-project/lotus#7546](https://github.com/filecoin-project/lotus/pull/7546))
+- Expose per-state sector counts on the prometheus endpoint ([filecoin-project/lotus#7541](https://github.com/filecoin-project/lotus/pull/7541))
+- Add storage-id flag to proving check ([filecoin-project/lotus#7479](https://github.com/filecoin-project/lotus/pull/7479))
+- FilecoinEC: Improve a log message ([filecoin-project/lotus#7499](https://github.com/filecoin-project/lotus/pull/7499))
+- itests: retry deal when control addr is out of funds ([filecoin-project/lotus#7454](https://github.com/filecoin-project/lotus/pull/7454))
+- Normlize selector use within lotus ([filecoin-project/lotus#7467](https://github.com/filecoin-project/lotus/pull/7467))
+- sealing: Improve scheduling of ready work ([filecoin-project/lotus#7335](https://github.com/filecoin-project/lotus/pull/7335))
+- Remove dead example code + dep ([filecoin-project/lotus#7466](https://github.com/filecoin-project/lotus/pull/7466))
+
+## Bug Fixes
+- fix  the withdrawn amount unit ([filecoin-project/lotus#7563](https://github.com/filecoin-project/lotus/pull/7563))
+- rename vm#make{=>Account}Actor(). ([filecoin-project/lotus#7562](https://github.com/filecoin-project/lotus/pull/7562))
+- Fix used sector space accounting after AddPieceFailed ([filecoin-project/lotus#7530](https://github.com/filecoin-project/lotus/pull/7530))
+- Don't remove sector data when moving data into a shared path ([filecoin-project/lotus#7494](https://github.com/filecoin-project/lotus/pull/7494))
+- fix: support node instantiation in external packages ([filecoin-project/lotus#7511](https://github.com/filecoin-project/lotus/pull/7511))
+- Stop adding Jennifer's $HOME to lotus docs ([filecoin-project/lotus#7477](https://github.com/filecoin-project/lotus/pull/7477))
+- Bugfix: Use correct startup network versions ([filecoin-project/lotus#7486](https://github.com/filecoin-project/lotus/pull/7486))
+- Dep upgrade pass ([filecoin-project/lotus#7478](https://github.com/filecoin-project/lotus/pull/7478))
+- Remove obsolete GS testplan - it now lives in go-graphsync ([filecoin-project/lotus#7469](https://github.com/filecoin-project/lotus/pull/7469))
+- sealing: Recover sectors after failed AddPiece ([filecoin-project/lotus#7444](https://github.com/filecoin-project/lotus/pull/7444))
+
+## Dependency Updates
+- Update go-graphsync v0.10.1 ([filecoin-project/lotus#7457](https://github.com/filecoin-project/lotus/pull/7457))
+- update to proof v10.1.0 ([filecoin-project/lotus#7564](https://github.com/filecoin-project/lotus/pull/7564))
+- github.com/filecoin-project/specs-actors/v6 (v6.0.0 -> v6.0.1):
+- github.com/filecoin-project/go-jsonrpc (v0.1.4-0.20210217175800-45ea43ac2bec -> v0.1.5):
+- github.com/filecoin-project/go-fil-markets (v1.13.1 -> v1.13.3):
+- github.com/filecoin-project/go-data-transfer (v1.11.1 -> v1.11.4):
+- github.com/filecoin-project/go-crypto (v0.0.0-20191218222705-effae4ea9f03 -> v0.0.1):
+- github.com/filecoin-project/go-commp-utils (v0.1.1-0.20210427191551-70bf140d31c7 -> v0.1.2):
+- github.com/filecoin-project/go-cbor-util (v0.0.0-20191219014500-08c40a1e63a2 -> v0.0.1):
+- github.com/filecoin-project/go-address (v0.0.5 -> v0.0.6):
+- unpin the yamux dependency ([filecoin-project/lotus#7532](https://github.com/filecoin-project/lotus/pull/7532)
+- peerstore@v0.2.9 was withdrawn, let's not depend on it directly ([filecoin-project/lotus#7481](https://github.com/filecoin-project/lotus/pull/7481))
+- chore(deps): use tagged github.com/ipld/go-ipld-selector-text-lite ([filecoin-project/lotus#7464](https://github.com/filecoin-project/lotus/pull/7464))
+- Stop indirectly depending on deprecated github.com/prometheus/common ([filecoin-project/lotus#7473](https://github.com/filecoin-project/lotus/pull/7473))
+
+## Others
+- fix the changelog ([filecoin-project/lotus#7594](https://github.com/filecoin-project/lotus/pull/7594))
+- v1.13.1-rc2 prep ([filecoin-project/lotus#7593](https://github.com/filecoin-project/lotus/pull/7593))
+- lotus v1.13.1-rc1 ([filecoin-project/lotus#7569](https://github.com/filecoin-project/lotus/pull/7569))
+- misc: back-port v1.13.0 back to master ([filecoin-project/lotus#7537](https://github.com/filecoin-project/lotus/pull/7537))
+- Inline codegen ([filecoin-project/lotus#7495](https://github.com/filecoin-project/lotus/pull/7495))
+- releases -> master ([filecoin-project/lotus#7507](https://github.com/filecoin-project/lotus/pull/7507))
+- Make chocolate back to master ([filecoin-project/lotus#7493](https://github.com/filecoin-project/lotus/pull/7493))
+- restore filters for the build-macos job ([filecoin-project/lotus#7455](https://github.com/filecoin-project/lotus/pull/7455))
+- bump master to v1.13.1-dev ([filecoin-project/lotus#7451](https://github.com/filecoin-project/lotus/pull/7451))
+
+Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| @magik6k | 27 | +1285/-531 | 76 |
+| @ribasushi | 7 | +265/-1635 | 21 |
+| @raulk | 2 | +2/-737 | 13 |
+| @nonsens | 4 | +391/-21 | 19 |
+| @arajasek | 6 | +216/-23 | 14 |
+| @jennijuju| 8 | +102/-37 | 29 |
+| Steven Allen | 2 | +77/-29 | 6 |
+| @jennijuju | 4 | +19/-18 | 11 |
+| @dirkmc | 2 | +9/-9 | 4 |
+| @@coryschwartz | 1 | +16/-2 | 2 |
+| @frrist | 1 | +12/-0 | 2 |
+| @Kubuxu | 5 | +5/-5 | 5 |
+| @hunjixin | 2 | +6/-3 | 2 |
+| @vyzo | 1 | +3/-3 | 2 |
+| @@rvagg | 1 | +3/-3 | 2 |
+| @hannahhoward | 1 | +3/-2 | 2 |
+| Marten Seemann | 1 | +3/-0 | 1 |
+| @ZenGround0 | 1 | +1/-1 | 1 |
+ 
+
 # v1.13.0 / 2021-10-18

 Lotus v1.13.0 is a *highly recommended* feature release for all lotus users(i.e: storage providers, data brokers, application developers and so on) that supports the upcoming 
@@ -144,6 +329,9 @@ Note that this release is built on top of lotus v1.11.3. Enterprising users like
 - Extend FaultMaxAge to 6 weeks for actors v6 on test networks only ([filecoin-project/lotus#7421](https://github.com/filecoin-project/lotus/pull/7421))

 ## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
 | @ZenGround0 | 12 | +4202/-2752 | 187 |
 | @arajasek | 25 | +4567/-854 | 190 |
 | @laudiacay | 4 | +1276/-435 | 37 |
@@ -36,7 +36,7 @@ WORKDIR /opt/filecoin
 ARG RUSTFLAGS=""
 ARG GOFLAGS=""

-RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway
+RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway lotus-stats


 FROM ubuntu:20.04 AS base
@@ -66,8 +66,6 @@ COPY scripts/docker-lotus-entrypoint.sh /

 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
 ENV LOTUS_PATH /var/lib/lotus
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
 ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
 ENV DOCKER_LOTUS_IMPORT_WALLET ""

@@ -92,8 +90,6 @@ MAINTAINER Lotus Development Team
 COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/

 ENV WALLET_PATH /var/lib/lotus-wallet
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831

 RUN mkdir /var/lib/lotus-wallet
 RUN chown fc: /var/lib/lotus-wallet
@@ -114,10 +110,6 @@ MAINTAINER Lotus Development Team

 COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/

-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
-
 USER fc

 EXPOSE 1234
@@ -135,11 +127,7 @@ COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
 COPY scripts/docker-lotus-miner-entrypoint.sh /

 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
 ENV LOTUS_MINER_PATH /var/lib/lotus-miner
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
-ENV DOCKER_LOTUS_MINER_INIT true

 RUN mkdir /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
 RUN chown fc: /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
@@ -163,10 +151,7 @@ MAINTAINER Lotus Development Team
 COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/

 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
 ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831

 RUN mkdir /var/lib/lotus-worker
 RUN chown fc: /var/lib/lotus-worker
@@ -186,16 +171,11 @@ CMD ["-help"]
 from base as lotus-all-in-one

 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
-ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
-ENV LOTUS_JAEGER_AGENT_PORT 6831
 ENV LOTUS_MINER_PATH /var/lib/lotus-miner
 ENV LOTUS_PATH /var/lib/lotus
 ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
-ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
 ENV WALLET_PATH /var/lib/lotus-wallet
 ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
-ENV DOCKER_LOTUS_MINER_INIT true

 COPY --from=builder /opt/filecoin/lotus      /usr/local/bin/
 COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
@@ -203,6 +183,7 @@ COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
 COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
 COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
 COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
+COPY --from=builder /opt/filecoin/lotus-stats /usr/local/bin/

 RUN mkdir /var/tmp/filecoin-proof-parameters
 RUN mkdir /var/lib/lotus
@@ -291,6 +291,7 @@ method-gen: api-gen
 	(cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go)

 actors-gen:
+	$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
 	$(GOCC) run ./chain/actors/agen
 	$(GOCC) fmt ./...

@@ -7,7 +7,6 @@ import (
 	"time"

 	"github.com/ipfs/go-cid"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	"github.com/libp2p/go-libp2p-core/peer"

 	"github.com/filecoin-project/go-address"
@@ -28,7 +27,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/types"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
 )
@@ -291,7 +289,7 @@ type FullNode interface {

 	// // UX ?

-	// MethodGroup: Wallet
+	// MethodGroup: WalletF

 	// WalletNew creates a new address in the wallet with the given sigType.
 	// Available key types: bls, secp256k1, secp256k1-ledger
@@ -352,10 +350,11 @@ type FullNode interface {
 	// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
 	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error) //perm:read
 	// ClientRetrieve initiates the retrieval of a file, as specified in the order.
-	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error //perm:admin
-	// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
-	// of status updates.
-	ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
+	ClientRetrieve(ctx context.Context, params RetrievalOrder) (*RestrievalRes, error) //perm:admin
+	// ClientRetrieveWait waits for retrieval to be complete
+	ClientRetrieveWait(ctx context.Context, deal retrievalmarket.DealID) error //perm:admin
+	// ClientExport exports a file stored in the local filestore to a system file
+	ClientExport(ctx context.Context, exportRef ExportRef, fileRef FileRef) error //perm:admin
 	// ClientListRetrievals returns information about retrievals made by the local client
 	ClientListRetrievals(ctx context.Context) ([]RetrievalInfo, error) //perm:write
 	// ClientGetRetrievalUpdates returns status of updated retrieval deals
@@ -630,10 +629,14 @@ type FullNode interface {
 	// <sender address of the approve msg>, <method to call in the proposed message>, <params to include in the proposed message>
 	MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign

+	// MsigCancel cancels a previously-proposed multisig message
+	// It takes the following params: <multisig address>, <proposed transaction ID> <signer address>
+	MsigCancel(context.Context, address.Address, uint64, address.Address) (*MessagePrototype, error) //perm:sign
+
 	// MsigCancel cancels a previously-proposed multisig message
 	// It takes the following params: <multisig address>, <proposed transaction ID>, <recipient address>, <value to transfer>,
 	// <sender address of the cancel msg>, <method to call in the proposed message>, <params to include in the proposed message>
-	MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign
+	MsigCancelTxnHash(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign

 	// MsigAddPropose proposes adding a signer in the multisig
 	// It takes the following params: <multisig address>, <sender address of the propose msg>,
@@ -930,15 +933,14 @@ type MarketDeal struct {
 }

 type RetrievalOrder struct {
-	// TODO: make this less unixfs specific
-	Root                  cid.Cid
-	Piece                 *cid.Cid
-	DatamodelPathSelector *textselector.Expression
-	Size                  uint64
+	Root         cid.Cid
+	Piece        *cid.Cid
+	DataSelector *Selector
+
+	// todo: Size/Total are only used for calculating price per byte; we should let users just pass that
+	Size  uint64
+	Total types.BigInt

-	FromLocalCAR string // if specified, get data from a local CARv2 file.
-	// TODO: support offset
-	Total                   types.BigInt
 	UnsealPrice             types.BigInt
 	PaymentInterval         uint64
 	PaymentIntervalIncrease uint64
@@ -31,6 +31,8 @@ import (
 type Gateway interface {
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
 	ChainHead(ctx context.Context) (*types.TipSet, error)
+	ChainGetParentMessages(context.Context, cid.Cid) ([]Message, error)
+	ChainGetParentReceipts(context.Context, cid.Cid) ([]*types.MessageReceipt, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
 	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
 	ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*HeadChange, error)
@@ -39,6 +41,7 @@ type Gateway interface {
 	ChainGetTipSetAfterHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
 	ChainNotify(context.Context) (<-chan []*HeadChange, error)
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainGetGenesis(context.Context) (*types.TipSet, error)
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
 	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
@@ -166,6 +166,7 @@ type StorageMiner interface {
 	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
 	MarketPendingDeals(ctx context.Context) (PendingDealInfo, error)                                                             //perm:write
 	MarketPublishPendingDeals(ctx context.Context) error                                                                         //perm:admin
+	MarketRetryPublishDeal(ctx context.Context, propcid cid.Cid) error                                                           //perm:admin

 	// DagstoreListShards returns information about all shards known to the
 	// DAG store. Only available on nodes running the markets subsystem.
@@ -15,7 +15,6 @@ import (
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-filestore"
 	"github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -24,7 +23,7 @@ import (
 	"github.com/multiformats/go-multiaddr"

 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	filestore2 "github.com/filecoin-project/go-fil-markets/filestore"
+	filestore "github.com/filecoin-project/go-fil-markets/filestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	textselector "github.com/ipld/go-ipld-selector-text-lite"
@@ -92,6 +91,8 @@ func init() {

 	storeIDExample := imports.ID(50)
 	textSelExample := textselector.Expression("Links/21/Hash/Links/42/Hash")
+	apiSelExample := api.Selector("Links/21/Hash/Links/42/Hash")
+	clientEvent := retrievalmarket.ClientEventDealAccepted

 	addExample(bitfield.NewFromSet([]uint64{5}))
 	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
@@ -110,7 +111,6 @@ func init() {
 	addExample(abi.UnpaddedPieceSize(1024))
 	addExample(abi.UnpaddedPieceSize(1024).Padded())
 	addExample(abi.DealID(5432))
-	addExample(filestore.StatusFileChanged)
 	addExample(abi.SectorNumber(9))
 	addExample(abi.SectorSize(32 * 1024 * 1024 * 1024))
 	addExample(api.MpoolChange(0))
@@ -124,9 +124,12 @@ func init() {
 	addExample(datatransfer.Ongoing)
 	addExample(storeIDExample)
 	addExample(&storeIDExample)
+	addExample(clientEvent)
+	addExample(&clientEvent)
 	addExample(retrievalmarket.ClientEventDealAccepted)
 	addExample(retrievalmarket.DealStatusNew)
 	addExample(&textSelExample)
+	addExample(&apiSelExample)
 	addExample(network.ReachabilityPublic)
 	addExample(build.NewestNetworkVersion)
 	addExample(map[string]int{"name": 42})
@@ -178,7 +181,7 @@ func init() {
 	ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr

 	// miner specific
-	addExample(filestore2.Path(".lotusminer/fstmp123"))
+	addExample(filestore.Path(".lotusminer/fstmp123"))
 	si := uint64(12)
 	addExample(&si)
 	addExample(retrievalmarket.DealID(5))
@@ -228,16 +231,18 @@ func init() {
 				Hostname: "host",
 				Resources: storiface.WorkerResources{
 					MemPhysical: 256 << 30,
+					MemUsed:     2 << 30,
 					MemSwap:     120 << 30,
-					MemReserved: 2 << 30,
+					MemSwapUsed: 2 << 30,
 					CPUs:        64,
 					GPUs:        []string{"aGPU 1337"},
+					Resources:   storiface.ResourceTable,
 				},
 			},
 			Enabled:    true,
 			MemUsedMin: 0,
 			MemUsedMax: 0,
-			GpuUsed:    false,
+			GpuUsed:    0,
 			CpuUse:     0,
 		},
 	})
@@ -283,6 +288,7 @@ func init() {
 		State: "ShardStateAvailable",
 		Error: "<error>",
 	})
+	addExample(storiface.ResourceTable)
 }

 func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
@@ -25,7 +25,6 @@ import (
 	miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	types "github.com/filecoin-project/lotus/chain/types"
 	alerting "github.com/filecoin-project/lotus/journal/alerting"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
 	imports "github.com/filecoin-project/lotus/node/repo/imports"
 	miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
@@ -537,6 +536,20 @@ func (mr *MockFullNodeMockRecorder) ClientDealSize(arg0, arg1 interface{}) *gomo
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientDealSize", reflect.TypeOf((*MockFullNode)(nil).ClientDealSize), arg0, arg1)
 }

+// ClientExport mocks base method.
+func (m *MockFullNode) ClientExport(arg0 context.Context, arg1 api.ExportRef, arg2 api.FileRef) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ClientExport", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ClientExport indicates an expected call of ClientExport.
+func (mr *MockFullNodeMockRecorder) ClientExport(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientExport", reflect.TypeOf((*MockFullNode)(nil).ClientExport), arg0, arg1, arg2)
+}
+
 // ClientFindData mocks base method.
 func (m *MockFullNode) ClientFindData(arg0 context.Context, arg1 cid.Cid, arg2 *cid.Cid) ([]api.QueryOffer, error) {
 	m.ctrl.T.Helper()
@@ -775,17 +788,18 @@ func (mr *MockFullNodeMockRecorder) ClientRestartDataTransfer(arg0, arg1, arg2,
 }

 // ClientRetrieve mocks base method.
-func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) error {
+func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder) (*api.RestrievalRes, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1, arg2)
-	ret0, _ := ret[0].(error)
-	return ret0
+	ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1)
+	ret0, _ := ret[0].(*api.RestrievalRes)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
 }

 // ClientRetrieve indicates an expected call of ClientRetrieve.
-func (mr *MockFullNodeMockRecorder) ClientRetrieve(arg0, arg1, arg2 interface{}) *gomock.Call {
+func (mr *MockFullNodeMockRecorder) ClientRetrieve(arg0, arg1 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieve", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieve), arg0, arg1, arg2)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieve", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieve), arg0, arg1)
 }

 // ClientRetrieveTryRestartInsufficientFunds mocks base method.
@@ -802,19 +816,18 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveTryRestartInsufficientFunds", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveTryRestartInsufficientFunds), arg0, arg1)
 }

-// ClientRetrieveWithEvents mocks base method.
-func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+// ClientRetrieveWait mocks base method.
+func (m *MockFullNode) ClientRetrieveWait(arg0 context.Context, arg1 retrievalmarket.DealID) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ClientRetrieveWithEvents", arg0, arg1, arg2)
-	ret0, _ := ret[0].(<-chan marketevents.RetrievalEvent)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
+	ret := m.ctrl.Call(m, "ClientRetrieveWait", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
 }

-// ClientRetrieveWithEvents indicates an expected call of ClientRetrieveWithEvents.
-func (mr *MockFullNodeMockRecorder) ClientRetrieveWithEvents(arg0, arg1, arg2 interface{}) *gomock.Call {
+// ClientRetrieveWait indicates an expected call of ClientRetrieveWait.
+func (mr *MockFullNodeMockRecorder) ClientRetrieveWait(arg0, arg1 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveWithEvents", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveWithEvents), arg0, arg1, arg2)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientRetrieveWait", reflect.TypeOf((*MockFullNode)(nil).ClientRetrieveWait), arg0, arg1)
 }

 // ClientStartDeal mocks base method.
@@ -1428,18 +1441,33 @@ func (mr *MockFullNodeMockRecorder) MsigApproveTxnHash(arg0, arg1, arg2, arg3, a
 }

 // MsigCancel mocks base method.
-func (m *MockFullNode) MsigCancel(arg0 context.Context, arg1 address.Address, arg2 uint64, arg3 address.Address, arg4 big.Int, arg5 address.Address, arg6 uint64, arg7 []byte) (*api.MessagePrototype, error) {
+func (m *MockFullNode) MsigCancel(arg0 context.Context, arg1 address.Address, arg2 uint64, arg3 address.Address) (*api.MessagePrototype, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "MsigCancel", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
+	ret := m.ctrl.Call(m, "MsigCancel", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(*api.MessagePrototype)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

 // MsigCancel indicates an expected call of MsigCancel.
-func (mr *MockFullNodeMockRecorder) MsigCancel(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call {
+func (mr *MockFullNodeMockRecorder) MsigCancel(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MsigCancel", reflect.TypeOf((*MockFullNode)(nil).MsigCancel), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MsigCancel", reflect.TypeOf((*MockFullNode)(nil).MsigCancel), arg0, arg1, arg2, arg3)
+}
+
+// MsigCancelTxnHash mocks base method.
+func (m *MockFullNode) MsigCancelTxnHash(arg0 context.Context, arg1 address.Address, arg2 uint64, arg3 address.Address, arg4 big.Int, arg5 address.Address, arg6 uint64, arg7 []byte) (*api.MessagePrototype, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MsigCancelTxnHash", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
+	ret0, _ := ret[0].(*api.MessagePrototype)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// MsigCancelTxnHash indicates an expected call of MsigCancelTxnHash.
+func (mr *MockFullNodeMockRecorder) MsigCancelTxnHash(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MsigCancelTxnHash", reflect.TypeOf((*MockFullNode)(nil).MsigCancelTxnHash), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
 }

 // MsigCreate mocks base method.
@@ -28,7 +28,6 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 	"github.com/filecoin-project/lotus/journal/alerting"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
 	"github.com/filecoin-project/specs-storage/storage"
@@ -162,6 +161,8 @@ type FullNodeStruct struct {

 		ClientDealSize func(p0 context.Context, p1 cid.Cid) (DataSize, error) `perm:"read"`

+		ClientExport func(p0 context.Context, p1 ExportRef, p2 FileRef) error `perm:"admin"`
+
 		ClientFindData func(p0 context.Context, p1 cid.Cid, p2 *cid.Cid) ([]QueryOffer, error) `perm:"read"`

 		ClientGenCar func(p0 context.Context, p1 FileRef, p2 string) error `perm:"write"`
@@ -194,11 +195,11 @@ type FullNodeStruct struct {

 		ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`

-		ClientRetrieve func(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) error `perm:"admin"`
+		ClientRetrieve func(p0 context.Context, p1 RetrievalOrder) (*RestrievalRes, error) `perm:"admin"`

 		ClientRetrieveTryRestartInsufficientFunds func(p0 context.Context, p1 address.Address) error `perm:"write"`

-		ClientRetrieveWithEvents func(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
+		ClientRetrieveWait func(p0 context.Context, p1 retrievalmarket.DealID) error `perm:"admin"`

 		ClientStartDeal func(p0 context.Context, p1 *StartDealParams) (*cid.Cid, error) `perm:"admin"`

@@ -270,7 +271,9 @@ type FullNodeStruct struct {

 		MsigApproveTxnHash func(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address, p4 address.Address, p5 types.BigInt, p6 address.Address, p7 uint64, p8 []byte) (*MessagePrototype, error) `perm:"sign"`

-		MsigCancel func(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address, p4 types.BigInt, p5 address.Address, p6 uint64, p7 []byte) (*MessagePrototype, error) `perm:"sign"`
+		MsigCancel func(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address) (*MessagePrototype, error) `perm:"sign"`
+
+		MsigCancelTxnHash func(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address, p4 types.BigInt, p5 address.Address, p6 uint64, p7 []byte) (*MessagePrototype, error) `perm:"sign"`

 		MsigCreate func(p0 context.Context, p1 uint64, p2 []address.Address, p3 abi.ChainEpoch, p4 types.BigInt, p5 address.Address, p6 types.BigInt) (*MessagePrototype, error) `perm:"sign"`

@@ -478,8 +481,14 @@ type GatewayStruct struct {
 	Internal struct {
 		ChainGetBlockMessages func(p0 context.Context, p1 cid.Cid) (*BlockMessages, error) ``

+		ChainGetGenesis func(p0 context.Context) (*types.TipSet, error) ``
+
 		ChainGetMessage func(p0 context.Context, p1 cid.Cid) (*types.Message, error) ``

+		ChainGetParentMessages func(p0 context.Context, p1 cid.Cid) ([]Message, error) ``
+
+		ChainGetParentReceipts func(p0 context.Context, p1 cid.Cid) ([]*types.MessageReceipt, error) ``
+
 		ChainGetPath func(p0 context.Context, p1 types.TipSetKey, p2 types.TipSetKey) ([]*HeadChange, error) ``

 		ChainGetTipSet func(p0 context.Context, p1 types.TipSetKey) (*types.TipSet, error) ``
@@ -683,6 +692,8 @@ type StorageMinerStruct struct {

 		MarketRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`

+		MarketRetryPublishDeal func(p0 context.Context, p1 cid.Cid) error `perm:"admin"`
+
 		MarketSetAsk func(p0 context.Context, p1 types.BigInt, p2 types.BigInt, p3 abi.ChainEpoch, p4 abi.PaddedPieceSize, p5 abi.PaddedPieceSize) error `perm:"admin"`

 		MarketSetRetrievalAsk func(p0 context.Context, p1 *retrievalmarket.Ask) error `perm:"admin"`
@@ -1347,6 +1358,17 @@ func (s *FullNodeStub) ClientDealSize(p0 context.Context, p1 cid.Cid) (DataSize,
 	return *new(DataSize), ErrNotSupported
 }

+func (s *FullNodeStruct) ClientExport(p0 context.Context, p1 ExportRef, p2 FileRef) error {
+	if s.Internal.ClientExport == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.ClientExport(p0, p1, p2)
+}
+
+func (s *FullNodeStub) ClientExport(p0 context.Context, p1 ExportRef, p2 FileRef) error {
+	return ErrNotSupported
+}
+
 func (s *FullNodeStruct) ClientFindData(p0 context.Context, p1 cid.Cid, p2 *cid.Cid) ([]QueryOffer, error) {
 	if s.Internal.ClientFindData == nil {
 		return *new([]QueryOffer), ErrNotSupported
@@ -1523,15 +1545,15 @@ func (s *FullNodeStub) ClientRestartDataTransfer(p0 context.Context, p1 datatran
 	return ErrNotSupported
 }

-func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) error {
+func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 RetrievalOrder) (*RestrievalRes, error) {
 	if s.Internal.ClientRetrieve == nil {
-		return ErrNotSupported
+		return nil, ErrNotSupported
 	}
-	return s.Internal.ClientRetrieve(p0, p1, p2)
+	return s.Internal.ClientRetrieve(p0, p1)
 }

-func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) error {
-	return ErrNotSupported
+func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 RetrievalOrder) (*RestrievalRes, error) {
+	return nil, ErrNotSupported
 }

 func (s *FullNodeStruct) ClientRetrieveTryRestartInsufficientFunds(p0 context.Context, p1 address.Address) error {
@@ -1545,15 +1567,15 @@ func (s *FullNodeStub) ClientRetrieveTryRestartInsufficientFunds(p0 context.Cont
 	return ErrNotSupported
 }

-func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) (<-chan marketevents.RetrievalEvent, error) {
-	if s.Internal.ClientRetrieveWithEvents == nil {
-		return nil, ErrNotSupported
+func (s *FullNodeStruct) ClientRetrieveWait(p0 context.Context, p1 retrievalmarket.DealID) error {
+	if s.Internal.ClientRetrieveWait == nil {
+		return ErrNotSupported
 	}
-	return s.Internal.ClientRetrieveWithEvents(p0, p1, p2)
+	return s.Internal.ClientRetrieveWait(p0, p1)
 }

-func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *FileRef) (<-chan marketevents.RetrievalEvent, error) {
-	return nil, ErrNotSupported
+func (s *FullNodeStub) ClientRetrieveWait(p0 context.Context, p1 retrievalmarket.DealID) error {
+	return ErrNotSupported
 }

 func (s *FullNodeStruct) ClientStartDeal(p0 context.Context, p1 *StartDealParams) (*cid.Cid, error) {
@@ -1941,14 +1963,25 @@ func (s *FullNodeStub) MsigApproveTxnHash(p0 context.Context, p1 address.Address
 	return nil, ErrNotSupported
 }

-func (s *FullNodeStruct) MsigCancel(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address, p4 types.BigInt, p5 address.Address, p6 uint64, p7 []byte) (*MessagePrototype, error) {
+func (s *FullNodeStruct) MsigCancel(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address) (*MessagePrototype, error) {
 	if s.Internal.MsigCancel == nil {
 		return nil, ErrNotSupported
 	}
-	return s.Internal.MsigCancel(p0, p1, p2, p3, p4, p5, p6, p7)
+	return s.Internal.MsigCancel(p0, p1, p2, p3)
 }

-func (s *FullNodeStub) MsigCancel(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address, p4 types.BigInt, p5 address.Address, p6 uint64, p7 []byte) (*MessagePrototype, error) {
+func (s *FullNodeStub) MsigCancel(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address) (*MessagePrototype, error) {
+	return nil, ErrNotSupported
+}
+
+func (s *FullNodeStruct) MsigCancelTxnHash(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address, p4 types.BigInt, p5 address.Address, p6 uint64, p7 []byte) (*MessagePrototype, error) {
+	if s.Internal.MsigCancelTxnHash == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.MsigCancelTxnHash(p0, p1, p2, p3, p4, p5, p6, p7)
+}
+
+func (s *FullNodeStub) MsigCancelTxnHash(p0 context.Context, p1 address.Address, p2 uint64, p3 address.Address, p4 types.BigInt, p5 address.Address, p6 uint64, p7 []byte) (*MessagePrototype, error) {
 	return nil, ErrNotSupported
 }

@@ -3030,6 +3063,17 @@ func (s *GatewayStub) ChainGetBlockMessages(p0 context.Context, p1 cid.Cid) (*Bl
 	return nil, ErrNotSupported
 }

+func (s *GatewayStruct) ChainGetGenesis(p0 context.Context) (*types.TipSet, error) {
+	if s.Internal.ChainGetGenesis == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.ChainGetGenesis(p0)
+}
+
+func (s *GatewayStub) ChainGetGenesis(p0 context.Context) (*types.TipSet, error) {
+	return nil, ErrNotSupported
+}
+
 func (s *GatewayStruct) ChainGetMessage(p0 context.Context, p1 cid.Cid) (*types.Message, error) {
 	if s.Internal.ChainGetMessage == nil {
 		return nil, ErrNotSupported
@@ -3041,6 +3085,28 @@ func (s *GatewayStub) ChainGetMessage(p0 context.Context, p1 cid.Cid) (*types.Me
 	return nil, ErrNotSupported
 }

+func (s *GatewayStruct) ChainGetParentMessages(p0 context.Context, p1 cid.Cid) ([]Message, error) {
+	if s.Internal.ChainGetParentMessages == nil {
+		return *new([]Message), ErrNotSupported
+	}
+	return s.Internal.ChainGetParentMessages(p0, p1)
+}
+
+func (s *GatewayStub) ChainGetParentMessages(p0 context.Context, p1 cid.Cid) ([]Message, error) {
+	return *new([]Message), ErrNotSupported
+}
+
+func (s *GatewayStruct) ChainGetParentReceipts(p0 context.Context, p1 cid.Cid) ([]*types.MessageReceipt, error) {
+	if s.Internal.ChainGetParentReceipts == nil {
+		return *new([]*types.MessageReceipt), ErrNotSupported
+	}
+	return s.Internal.ChainGetParentReceipts(p0, p1)
+}
+
+func (s *GatewayStub) ChainGetParentReceipts(p0 context.Context, p1 cid.Cid) ([]*types.MessageReceipt, error) {
+	return *new([]*types.MessageReceipt), ErrNotSupported
+}
+
 func (s *GatewayStruct) ChainGetPath(p0 context.Context, p1 types.TipSetKey, p2 types.TipSetKey) ([]*HeadChange, error) {
 	if s.Internal.ChainGetPath == nil {
 		return *new([]*HeadChange), ErrNotSupported
@@ -4020,6 +4086,17 @@ func (s *StorageMinerStub) MarketRestartDataTransfer(p0 context.Context, p1 data
 	return ErrNotSupported
 }

+func (s *StorageMinerStruct) MarketRetryPublishDeal(p0 context.Context, p1 cid.Cid) error {
+	if s.Internal.MarketRetryPublishDeal == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.MarketRetryPublishDeal(p0, p1)
+}
+
+func (s *StorageMinerStub) MarketRetryPublishDeal(p0 context.Context, p1 cid.Cid) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) MarketSetAsk(p0 context.Context, p1 types.BigInt, p2 types.BigInt, p3 abi.ChainEpoch, p4 abi.PaddedPieceSize, p5 abi.PaddedPieceSize) error {
 	if s.Internal.MarketSetAsk == nil {
 		return ErrNotSupported
@@ -5,11 +5,10 @@ import (
 	"fmt"
 	"time"

-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/lotus/chain/types"
-
 	datatransfer "github.com/filecoin-project/go-data-transfer"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/ipfs/go-cid"

 	"github.com/libp2p/go-libp2p-core/peer"
@@ -194,4 +193,47 @@ type RetrievalInfo struct {

 	TransferChannelID *datatransfer.ChannelID
 	DataTransfer      *DataTransferChannel
+
+	// optional event if part of ClientGetRetrievalUpdates
+	Event *retrievalmarket.ClientEvent
+}
+
+type RestrievalRes struct {
+	DealID retrievalmarket.DealID
+}
+
+// Selector specifies ipld selector string
+// - if the string starts with '{', it's interpreted as json selector string
+//   see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
+// - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
+//   see https://github.com/ipld/go-ipld-selector-text-lite
+type Selector string
+
+type DagSpec struct {
+	// DataSelector matches data to be retrieved
+	// - when using textselector, the path specifies subtree
+	// - the matched graph must have a single root
+	DataSelector *Selector
+
+	// ExportMerkleProof is applicable only when exporting to a CAR file via a path textselector
+	// When true, in addition to the selection target, the resulting CAR will contain every block along the
+	// path back to, and including the original root
+	// When false the resulting CAR contains only the blocks of the target subdag
+	ExportMerkleProof bool
+}
+
+type ExportRef struct {
+	Root cid.Cid
+
+	// DAGs array specifies a list of DAGs to export
+	// - If exporting into unixfs files, only one DAG is supported, DataSelector is only used to find the targeted root node
+	// - If exporting into a car file
+	//   - When exactly one text-path DataSelector is specified exports the subgraph and its full merkle-path from the original root
+	//   - Otherwise ( multiple paths and/or JSON selector specs) determines each individual subroot and exports the subtrees as a multi-root car
+	// - When not specified defaults to a single DAG:
+	//   - Data - the entire DAG: `{"R":{"l":{"none":{}},":>":{"a":{">":{"@":{}}}}}}`
+	DAGs []DagSpec
+
+	FromLocalCAR string // if specified, get data from a local CARv2 file.
+	DealID       retrievalmarket.DealID
 }
@@ -12,6 +12,7 @@ import (
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
 	"github.com/ipfs/go-cid"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	"github.com/libp2p/go-libp2p-core/peer"

 	"github.com/filecoin-project/lotus/api"
@@ -325,10 +326,10 @@ type FullNode interface {
 	// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
 	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) //perm:read
 	// ClientRetrieve initiates the retrieval of a file, as specified in the order.
-	ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error //perm:admin
+	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error //perm:admin
 	// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
 	// of status updates.
-	ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
+	ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
 	// ClientQueryAsk returns a signed StorageAsk from the specified miner.
 	// ClientListRetrievals returns information about retrievals made by the local client
 	ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) //perm:write
@@ -714,3 +715,37 @@ type FullNode interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 }
+
+func OfferOrder(o api.QueryOffer, client address.Address) RetrievalOrder {
+	return RetrievalOrder{
+		Root:                    o.Root,
+		Piece:                   o.Piece,
+		Size:                    o.Size,
+		Total:                   o.MinPrice,
+		UnsealPrice:             o.UnsealPrice,
+		PaymentInterval:         o.PaymentInterval,
+		PaymentIntervalIncrease: o.PaymentIntervalIncrease,
+		Client:                  client,
+
+		Miner:     o.Miner,
+		MinerPeer: &o.MinerPeer,
+	}
+}
+
+type RetrievalOrder struct {
+	// TODO: make this less unixfs specific
+	Root                  cid.Cid
+	Piece                 *cid.Cid
+	DatamodelPathSelector *textselector.Expression
+	Size                  uint64
+
+	FromLocalCAR string // if specified, get data from a local CARv2 file.
+	// TODO: support offset
+	Total                   types.BigInt
+	UnsealPrice             types.BigInt
+	PaymentInterval         uint64
+	PaymentIntervalIncrease uint64
+	Client                  address.Address
+	Miner                   address.Address
+	MinerPeer               *retrievalmarket.RetrievalPeer
+}
@@ -125,11 +125,11 @@ type FullNodeStruct struct {

 		ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`

-		ClientRetrieve func(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error `perm:"admin"`
+		ClientRetrieve func(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error `perm:"admin"`

 		ClientRetrieveTryRestartInsufficientFunds func(p0 context.Context, p1 address.Address) error `perm:"write"`

-		ClientRetrieveWithEvents func(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
+		ClientRetrieveWithEvents func(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`

 		ClientStartDeal func(p0 context.Context, p1 *api.StartDealParams) (*cid.Cid, error) `perm:"admin"`

@@ -965,14 +965,14 @@ func (s *FullNodeStub) ClientRestartDataTransfer(p0 context.Context, p1 datatran
 	return ErrNotSupported
 }

-func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error {
+func (s *FullNodeStruct) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error {
 	if s.Internal.ClientRetrieve == nil {
 		return ErrNotSupported
 	}
 	return s.Internal.ClientRetrieve(p0, p1, p2)
 }

-func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) error {
+func (s *FullNodeStub) ClientRetrieve(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) error {
 	return ErrNotSupported
 }

@@ -987,14 +987,14 @@ func (s *FullNodeStub) ClientRetrieveTryRestartInsufficientFunds(p0 context.Cont
 	return ErrNotSupported
 }

-func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+func (s *FullNodeStruct) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
 	if s.Internal.ClientRetrieveWithEvents == nil {
 		return nil, ErrNotSupported
 	}
 	return s.Internal.ClientRetrieveWithEvents(p0, p1, p2)
 }

-func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 api.RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+func (s *FullNodeStub) ClientRetrieveWithEvents(p0 context.Context, p1 RetrievalOrder, p2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
 	return nil, ErrNotSupported
 }

@@ -21,6 +21,7 @@ import (
 	network "github.com/filecoin-project/go-state-types/network"
 	api "github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
+	v0api "github.com/filecoin-project/lotus/api/v0api"
 	miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	types "github.com/filecoin-project/lotus/chain/types"
 	alerting "github.com/filecoin-project/lotus/journal/alerting"
@@ -760,7 +761,7 @@ func (mr *MockFullNodeMockRecorder) ClientRestartDataTransfer(arg0, arg1, arg2,
 }

 // ClientRetrieve mocks base method.
-func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) error {
+func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) error {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1, arg2)
 	ret0, _ := ret[0].(error)
@@ -788,7 +789,7 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar
 }

 // ClientRetrieveWithEvents mocks base method.
-func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "ClientRetrieveWithEvents", arg0, arg1, arg2)
 	ret0, _ := ret[0].(<-chan marketevents.RetrievalEvent)
@@ -3,7 +3,10 @@ package v0api
 import (
 	"context"

+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
+	marketevents "github.com/filecoin-project/lotus/markets/loggers"

 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -108,7 +111,7 @@ func (w *WrapperV1Full) MsigApproveTxnHash(ctx context.Context, msig address.Add
 }

 func (w *WrapperV1Full) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
-	p, err := w.FullNode.MsigCancel(ctx, msig, txID, to, amt, src, method, params)
+	p, err := w.FullNode.MsigCancelTxnHash(ctx, msig, txID, to, amt, src, method, params)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
 	}
@@ -194,4 +197,144 @@ func (w *WrapperV1Full) ChainGetRandomnessFromBeacon(ctx context.Context, tsk ty
 	return w.StateGetRandomnessFromBeacon(ctx, personalization, randEpoch, entropy, tsk)
 }

+func (w *WrapperV1Full) ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error {
+	events := make(chan marketevents.RetrievalEvent)
+	go w.clientRetrieve(ctx, order, ref, events)
+
+	for {
+		select {
+		case evt, ok := <-events:
+			if !ok { // done successfully
+				return nil
+			}
+
+			if evt.Err != "" {
+				return xerrors.Errorf("retrieval failed: %s", evt.Err)
+			}
+		case <-ctx.Done():
+			return xerrors.Errorf("retrieval timed out")
+		}
+	}
+}
+
+func (w *WrapperV1Full) ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
+	events := make(chan marketevents.RetrievalEvent)
+	go w.clientRetrieve(ctx, order, ref, events)
+	return events, nil
+}
+
+func readSubscribeEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents <-chan api.RetrievalInfo, events chan marketevents.RetrievalEvent) error {
+	for {
+		var subscribeEvent api.RetrievalInfo
+		var evt retrievalmarket.ClientEvent
+		select {
+		case <-ctx.Done():
+			return xerrors.New("Retrieval Timed Out")
+		case subscribeEvent = <-subscribeEvents:
+			if subscribeEvent.ID != dealID {
+				// we can't check the deal ID ahead of time because:
+				// 1. We need to subscribe before retrieving.
+				// 2. We won't know the deal ID until after retrieving.
+				continue
+			}
+			if subscribeEvent.Event != nil {
+				evt = *subscribeEvent.Event
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return xerrors.New("Retrieval Timed Out")
+		case events <- marketevents.RetrievalEvent{
+			Event:         evt,
+			Status:        subscribeEvent.Status,
+			BytesReceived: subscribeEvent.BytesReceived,
+			FundsSpent:    subscribeEvent.TotalPaid,
+		}:
+		}
+
+		switch subscribeEvent.Status {
+		case retrievalmarket.DealStatusCompleted:
+			return nil
+		case retrievalmarket.DealStatusRejected:
+			return xerrors.Errorf("Retrieval Proposal Rejected: %s", subscribeEvent.Message)
+		case
+			retrievalmarket.DealStatusDealNotFound,
+			retrievalmarket.DealStatusErrored:
+			return xerrors.Errorf("Retrieval Error: %s", subscribeEvent.Message)
+		}
+	}
+}
+
+func (w *WrapperV1Full) clientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef, events chan marketevents.RetrievalEvent) {
+	defer close(events)
+
+	finish := func(e error) {
+		if e != nil {
+			events <- marketevents.RetrievalEvent{Err: e.Error(), FundsSpent: big.Zero()}
+		}
+	}
+
+	var dealID retrievalmarket.DealID
+	if order.FromLocalCAR == "" {
+		// Subscribe to events before retrieving to avoid losing events.
+		subscribeCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		retrievalEvents, err := w.ClientGetRetrievalUpdates(subscribeCtx)
+
+		if err != nil {
+			finish(xerrors.Errorf("GetRetrievalUpdates failed: %w", err))
+			return
+		}
+
+		retrievalRes, err := w.FullNode.ClientRetrieve(ctx, api.RetrievalOrder{
+			Root:                    order.Root,
+			Piece:                   order.Piece,
+			Size:                    order.Size,
+			Total:                   order.Total,
+			UnsealPrice:             order.UnsealPrice,
+			PaymentInterval:         order.PaymentInterval,
+			PaymentIntervalIncrease: order.PaymentIntervalIncrease,
+			Client:                  order.Client,
+			Miner:                   order.Miner,
+			MinerPeer:               order.MinerPeer,
+		})
+
+		if err != nil {
+			finish(xerrors.Errorf("Retrieve failed: %w", err))
+			return
+		}
+
+		dealID = retrievalRes.DealID
+
+		err = readSubscribeEvents(ctx, retrievalRes.DealID, retrievalEvents, events)
+		if err != nil {
+			finish(xerrors.Errorf("Retrieve: %w", err))
+			return
+		}
+	}
+
+	// If ref is nil, it only fetches the data into the configured blockstore.
+	if ref == nil {
+		finish(nil)
+		return
+	}
+
+	eref := api.ExportRef{
+		Root:         order.Root,
+		FromLocalCAR: order.FromLocalCAR,
+		DealID:       dealID,
+	}
+
+	if order.DatamodelPathSelector != nil {
+		s := api.Selector(*order.DatamodelPathSelector)
+		eref.DAGs = append(eref.DAGs, api.DagSpec{
+			DataSelector:      &s,
+			ExportMerkleProof: true,
+		})
+	}
+
+	finish(w.ClientExport(ctx, eref, *ref))
+}
+
 var _ FullNode = &WrapperV1Full{}
@@ -58,7 +58,7 @@ var (
 	FullAPIVersion1 = newVer(2, 1, 0)

 	MinerAPIVersion0  = newVer(1, 2, 0)
-	WorkerAPIVersion0 = newVer(1, 1, 0)
+	WorkerAPIVersion0 = newVer(1, 5, 0)
 )

 //nolint:varcheck,deadcode
@@ -28,8 +28,16 @@ const UnixfsLinksPerLevel = 1024
 const AllowableClockDriftSecs = uint64(1)

 // TODO: This is still terrible...What's the impact of updating this before mainnet actually upgrades
+/* inline-gen template
+
+const NewestNetworkVersion = network.Version{{.latestNetworkVersion}}
+
+/* inline-gen start */
+
 const NewestNetworkVersion = network.Version14

+/* inline-gen end */
+
 // Epochs
 const ForkLengthThreshold = Finality

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }

 // BuildVersion is the local build version
-const BuildVersion = "1.13.0"
+const BuildVersion = "1.13.2"

 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {
@@ -67,3 +67,22 @@ func SealProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version) (abi.

 	return 0, xerrors.Errorf("unsupported network version")
 }
+
+// WindowPoStProofTypeFromSectorSize returns preferred post proof type for creating
+// new miner actors and new sectors
+func WindowPoStProofTypeFromSectorSize(ssize abi.SectorSize) (abi.RegisteredPoStProof, error) {
+	switch ssize {
+	case 2 << 10:
+		return abi.RegisteredPoStProof_StackedDrgWindow2KiBV1, nil
+	case 8 << 20:
+		return abi.RegisteredPoStProof_StackedDrgWindow8MiBV1, nil
+	case 512 << 20:
+		return abi.RegisteredPoStProof_StackedDrgWindow512MiBV1, nil
+	case 32 << 30:
+		return abi.RegisteredPoStProof_StackedDrgWindow32GiBV1, nil
+	case 64 << 30:
+		return abi.RegisteredPoStProof_StackedDrgWindow64GiBV1, nil
+	default:
+		return 0, xerrors.Errorf("unsupported sector size for miner: %v", ssize)
+	}
+}
@@ -8,9 +8,21 @@ import (

 type Version int

+/* inline-gen template
+
+var LatestVersion = {{.latestActorsVersion}}
+
+var Versions = []int{ {{range .actorVersions}} {{.}}, {{end}} }
+
+const ({{range .actorVersions}}
+	Version{{.}} Version = {{.}}{{end}}
+)
+
+/* inline-gen start */
+
 var LatestVersion = 6

-var Versions = []int{0, 2, 3, 4, 5, LatestVersion}
+var Versions = []int{0, 2, 3, 4, 5, 6}

 const (
 	Version0 Version = 0
@@ -21,6 +33,8 @@ const (
 	Version6 Version = 6
 )

+/* inline-gen end */
+
 // Converts a network version into an actors adt version.
 func VersionForNetwork(version network.Version) (Version, error) {
 	switch version {
@@ -16,6 +16,12 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"

+	/* inline-gen template
+	   {{range .actorVersions}}
+	   	exported{{.}} "github.com/filecoin-project/specs-actors{{import .}}actors/builtin/exported"{{end}}
+
+	/* inline-gen start */
+
 	exported0 "github.com/filecoin-project/specs-actors/actors/builtin/exported"
 	exported2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported"
 	exported3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/exported"
@@ -23,6 +29,8 @@ import (
 	exported5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/exported"
 	exported6 "github.com/filecoin-project/specs-actors/v6/actors/builtin/exported"

+	/* inline-gen end */
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -39,6 +47,11 @@ func NewActorRegistry() *vm.ActorRegistry {
 	inv := vm.NewActorRegistry()

 	// TODO: define all these properties on the actors themselves, in specs-actors.
+	/* inline-gen template
+	{{range .actorVersions}}
+	inv.Register(vm.ActorsVersionPredicate(actors.Version{{.}}), exported{{.}}.BuiltinActors()...){{end}}
+
+	/* inline-gen start */

 	inv.Register(vm.ActorsVersionPredicate(actors.Version0), exported0.BuiltinActors()...)
 	inv.Register(vm.ActorsVersionPredicate(actors.Version2), exported2.BuiltinActors()...)
@@ -47,6 +60,8 @@ func NewActorRegistry() *vm.ActorRegistry {
 	inv.Register(vm.ActorsVersionPredicate(actors.Version5), exported5.BuiltinActors()...)
 	inv.Register(vm.ActorsVersionPredicate(actors.Version6), exported6.BuiltinActors()...)

+	/* inline-gen end */
+
 	return inv
 }

@@ -823,7 +823,7 @@ func (filec *FilecoinEC) checkPowerAndGetWorkerKey(ctx context.Context, bh *type

 	key, err := stmgr.GetMinerWorkerRaw(ctx, filec.sm, lbst, bh.Miner)
 	if err != nil {
-		log.Warnf("failed to resolve worker key for miner %s: %s", bh.Miner, err)
+		log.Warnf("failed to resolve worker key for miner %s and block height %d: %s", bh.Miner, bh.Height, err)
 		return address.Undef, ErrSoftFailure
 	}

@@ -152,7 +152,16 @@ func VersionForNetwork(ver network.Version) (types.StateTreeVersion, error) {
 		return types.StateTreeVersion2, nil
 	case network.Version12:
 		return types.StateTreeVersion3, nil
+
+		/* inline-gen template
+		{{$lastNv := .latestNetworkVersion}}
+			case{{range .networkVersions}} {{if (ge . 13.)}} network.Version{{.}}{{if (lt . $lastNv)}},{{end}}{{end}}{{end}}:
+
+		/* inline-gen start */
+
 	case network.Version13, network.Version14:
+
+		/* inline-gen end */
 		return types.StateTreeVersion4, nil
 	default:
 		panic(fmt.Sprintf("unsupported network version %d", ver))
@@ -14,6 +14,12 @@ import (
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"

+	/* inline-gen template
+	   {{range .actorVersions}}
+	   	builtin{{.}} "github.com/filecoin-project/specs-actors{{import .}}actors/builtin"{{end}}
+
+	/* inline-gen start */
+
 	builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 	builtin3 "github.com/filecoin-project/specs-actors/v3/actors/builtin"
@@ -21,6 +27,8 @@ import (
 	builtin5 "github.com/filecoin-project/specs-actors/v5/actors/builtin"
 	builtin6 "github.com/filecoin-project/specs-actors/v6/actors/builtin"

+	/* inline-gen end */
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/actors/aerrors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -60,7 +68,7 @@ func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, add
 		return nil, address.Undef, aerrors.Escalate(err, "unsupported network version")
 	}

-	act, aerr := makeActor(av, addr)
+	act, aerr := makeAccountActor(av, addr)
 	if aerr != nil {
 		return nil, address.Undef, aerr
 	}
@@ -87,7 +95,7 @@ func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, add
 	return act, addrID, nil
 }

-func makeActor(ver actors.Version, addr address.Address) (*types.Actor, aerrors.ActorError) {
+func makeAccountActor(ver actors.Version, addr address.Address) (*types.Actor, aerrors.ActorError) {
 	switch addr.Protocol() {
 	case address.BLS, address.SECP256K1:
 		return newAccountActor(ver), nil
@@ -104,6 +112,12 @@ func newAccountActor(ver actors.Version) *types.Actor {
 	// TODO: ActorsUpgrade use a global actor registry?
 	var code cid.Cid
 	switch ver {
+	/* inline-gen template
+	   {{range .actorVersions}}
+		case actors.Version{{.}}:
+			code = builtin{{.}}.AccountActorCodeID{{end}}
+	/* inline-gen start */
+
 	case actors.Version0:
 		code = builtin0.AccountActorCodeID
 	case actors.Version2:
@@ -116,6 +130,7 @@ func newAccountActor(ver actors.Version) *types.Actor {
 		code = builtin5.AccountActorCodeID
 	case actors.Version6:
 		code = builtin6.AccountActorCodeID
+		/* inline-gen end */
 	default:
 		panic("unsupported actors version")
 	}
@@ -26,7 +26,6 @@ import (
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-cidutil/cidenc"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/multiformats/go-multibase"
 	"github.com/urfave/cli/v2"
@@ -94,6 +93,8 @@ var clientCmd = &cli.Command{
 		WithCategory("data", clientStat),
 		WithCategory("retrieval", clientFindCmd),
 		WithCategory("retrieval", clientRetrieveCmd),
+		WithCategory("retrieval", clientRetrieveCatCmd),
+		WithCategory("retrieval", clientRetrieveLsCmd),
 		WithCategory("retrieval", clientCancelRetrievalDealCmd),
 		WithCategory("retrieval", clientListRetrievalsCmd),
 		WithCategory("util", clientCommPCmd),
@@ -1029,209 +1030,6 @@ var clientFindCmd = &cli.Command{
 	},
 }

-const DefaultMaxRetrievePrice = "0.01"
-
-var clientRetrieveCmd = &cli.Command{
-	Name:      "retrieve",
-	Usage:     "Retrieve data from network",
-	ArgsUsage: "[dataCid outputPath]",
-	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:  "from",
-			Usage: "address to send transactions from",
-		},
-		&cli.BoolFlag{
-			Name:  "car",
-			Usage: "export to a car file instead of a regular file",
-		},
-		&cli.StringFlag{
-			Name:  "miner",
-			Usage: "miner address for retrieval, if not present it'll use local discovery",
-		},
-		&cli.StringFlag{
-			Name:  "datamodel-path-selector",
-			Usage: "a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal",
-		},
-		&cli.StringFlag{
-			Name:  "maxPrice",
-			Usage: fmt.Sprintf("maximum price the client is willing to consider (default: %s FIL)", DefaultMaxRetrievePrice),
-		},
-		&cli.StringFlag{
-			Name:  "pieceCid",
-			Usage: "require data to be retrieved from a specific Piece CID",
-		},
-		&cli.BoolFlag{
-			Name: "allow-local",
-		},
-	},
-	Action: func(cctx *cli.Context) error {
-		if cctx.NArg() != 2 {
-			return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments"))
-		}
-
-		fapi, closer, err := GetFullNodeAPI(cctx)
-		if err != nil {
-			return err
-		}
-		defer closer()
-		ctx := ReqContext(cctx)
-		afmt := NewAppFmt(cctx.App)
-
-		var payer address.Address
-		if cctx.String("from") != "" {
-			payer, err = address.NewFromString(cctx.String("from"))
-		} else {
-			payer, err = fapi.WalletDefaultAddress(ctx)
-		}
-		if err != nil {
-			return err
-		}
-
-		file, err := cid.Parse(cctx.Args().Get(0))
-		if err != nil {
-			return err
-		}
-
-		var pieceCid *cid.Cid
-		if cctx.String("pieceCid") != "" {
-			parsed, err := cid.Parse(cctx.String("pieceCid"))
-			if err != nil {
-				return err
-			}
-			pieceCid = &parsed
-		}
-
-		var order *lapi.RetrievalOrder
-		if cctx.Bool("allow-local") {
-			imports, err := fapi.ClientListImports(ctx)
-			if err != nil {
-				return err
-			}
-
-			for _, i := range imports {
-				if i.Root != nil && i.Root.Equals(file) {
-					order = &lapi.RetrievalOrder{
-						Root:         file,
-						FromLocalCAR: i.CARPath,
-
-						Total:       big.Zero(),
-						UnsealPrice: big.Zero(),
-					}
-					break
-				}
-			}
-		}
-
-		if order == nil {
-			var offer api.QueryOffer
-			minerStrAddr := cctx.String("miner")
-			if minerStrAddr == "" { // Local discovery
-				offers, err := fapi.ClientFindData(ctx, file, pieceCid)
-
-				var cleaned []api.QueryOffer
-				// filter out offers that errored
-				for _, o := range offers {
-					if o.Err == "" {
-						cleaned = append(cleaned, o)
-					}
-				}
-
-				offers = cleaned
-
-				// sort by price low to high
-				sort.Slice(offers, func(i, j int) bool {
-					return offers[i].MinPrice.LessThan(offers[j].MinPrice)
-				})
-				if err != nil {
-					return err
-				}
-
-				// TODO: parse offer strings from `client find`, make this smarter
-				if len(offers) < 1 {
-					fmt.Println("Failed to find file")
-					return nil
-				}
-				offer = offers[0]
-			} else { // Directed retrieval
-				minerAddr, err := address.NewFromString(minerStrAddr)
-				if err != nil {
-					return err
-				}
-				offer, err = fapi.ClientMinerQueryOffer(ctx, minerAddr, file, pieceCid)
-				if err != nil {
-					return err
-				}
-			}
-			if offer.Err != "" {
-				return fmt.Errorf("The received offer errored: %s", offer.Err)
-			}
-
-			maxPrice := types.MustParseFIL(DefaultMaxRetrievePrice)
-
-			if cctx.String("maxPrice") != "" {
-				maxPrice, err = types.ParseFIL(cctx.String("maxPrice"))
-				if err != nil {
-					return xerrors.Errorf("parsing maxPrice: %w", err)
-				}
-			}
-
-			if offer.MinPrice.GreaterThan(big.Int(maxPrice)) {
-				return xerrors.Errorf("failed to find offer satisfying maxPrice: %s", maxPrice)
-			}
-
-			o := offer.Order(payer)
-			order = &o
-		}
-		ref := &lapi.FileRef{
-			Path:  cctx.Args().Get(1),
-			IsCAR: cctx.Bool("car"),
-		}
-
-		if sel := textselector.Expression(cctx.String("datamodel-path-selector")); sel != "" {
-			order.DatamodelPathSelector = &sel
-		}
-
-		updates, err := fapi.ClientRetrieveWithEvents(ctx, *order, ref)
-		if err != nil {
-			return xerrors.Errorf("error setting up retrieval: %w", err)
-		}
-
-		var prevStatus retrievalmarket.DealStatus
-
-		for {
-			select {
-			case evt, ok := <-updates:
-				if ok {
-					afmt.Printf("> Recv: %s, Paid %s, %s (%s)\n",
-						types.SizeStr(types.NewInt(evt.BytesReceived)),
-						types.FIL(evt.FundsSpent),
-						retrievalmarket.ClientEvents[evt.Event],
-						retrievalmarket.DealStatuses[evt.Status],
-					)
-					prevStatus = evt.Status
-				}
-
-				if evt.Err != "" {
-					return xerrors.Errorf("retrieval failed: %s", evt.Err)
-				}
-
-				if !ok {
-					if prevStatus == retrievalmarket.DealStatusCompleted {
-						afmt.Println("Success")
-					} else {
-						afmt.Printf("saw final deal state %s instead of expected success state DealStatusCompleted\n",
-							retrievalmarket.DealStatuses[prevStatus])
-					}
-					return nil
-				}
-
-			case <-ctx.Done():
-				return xerrors.Errorf("retrieval timed out")
-			}
-		}
-	},
-}
-
 var clientListRetrievalsCmd = &cli.Command{
 	Name:  "list-retrievals",
 	Usage: "List retrieval market deals",
@@ -0,0 +1,602 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	"github.com/ipfs/go-merkledag"
+	carv2 "github.com/ipld/go-car/v2"
+	"github.com/ipld/go-car/v2/blockstore"
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/codec/dagjson"
+	basicnode "github.com/ipld/go-ipld-prime/node/basic"
+	"github.com/ipld/go-ipld-prime/traversal"
+	"github.com/ipld/go-ipld-prime/traversal/selector"
+	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
+	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
+	"github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr/net"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-state-types/big"
+
+	lapi "github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/markets/utils"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+const DefaultMaxRetrievePrice = "0"
+
+func retrieve(ctx context.Context, cctx *cli.Context, fapi lapi.FullNode, sel *lapi.Selector, printf func(string, ...interface{})) (*lapi.ExportRef, error) {
+	var payer address.Address
+	var err error
+	if cctx.String("from") != "" {
+		payer, err = address.NewFromString(cctx.String("from"))
+	} else {
+		payer, err = fapi.WalletDefaultAddress(ctx)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := cid.Parse(cctx.Args().Get(0))
+	if err != nil {
+		return nil, err
+	}
+
+	var pieceCid *cid.Cid
+	if cctx.String("pieceCid") != "" {
+		parsed, err := cid.Parse(cctx.String("pieceCid"))
+		if err != nil {
+			return nil, err
+		}
+		pieceCid = &parsed
+	}
+
+	var eref *lapi.ExportRef
+	if cctx.Bool("allow-local") {
+		imports, err := fapi.ClientListImports(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, i := range imports {
+			if i.Root != nil && i.Root.Equals(file) {
+				eref = &lapi.ExportRef{
+					Root:         file,
+					FromLocalCAR: i.CARPath,
+				}
+				break
+			}
+		}
+	}
+
+	// no local found, so make a retrieval
+	if eref == nil {
+		var offer lapi.QueryOffer
+		minerStrAddr := cctx.String("provider")
+		if minerStrAddr == "" { // Local discovery
+			offers, err := fapi.ClientFindData(ctx, file, pieceCid)
+
+			var cleaned []lapi.QueryOffer
+			// filter out offers that errored
+			for _, o := range offers {
+				if o.Err == "" {
+					cleaned = append(cleaned, o)
+				}
+			}
+
+			offers = cleaned
+
+			// sort by price low to high
+			sort.Slice(offers, func(i, j int) bool {
+				return offers[i].MinPrice.LessThan(offers[j].MinPrice)
+			})
+			if err != nil {
+				return nil, err
+			}
+
+			// TODO: parse offer strings from `client find`, make this smarter
+			if len(offers) < 1 {
+				fmt.Println("Failed to find file")
+				return nil, nil
+			}
+			offer = offers[0]
+		} else { // Directed retrieval
+			minerAddr, err := address.NewFromString(minerStrAddr)
+			if err != nil {
+				return nil, err
+			}
+			offer, err = fapi.ClientMinerQueryOffer(ctx, minerAddr, file, pieceCid)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if offer.Err != "" {
+			return nil, fmt.Errorf("offer error: %s", offer.Err)
+		}
+
+		maxPrice := types.MustParseFIL(DefaultMaxRetrievePrice)
+
+		if cctx.String("maxPrice") != "" {
+			maxPrice, err = types.ParseFIL(cctx.String("maxPrice"))
+			if err != nil {
+				return nil, xerrors.Errorf("parsing maxPrice: %w", err)
+			}
+		}
+
+		if offer.MinPrice.GreaterThan(big.Int(maxPrice)) {
+			return nil, xerrors.Errorf("failed to find offer satisfying maxPrice: %s", maxPrice)
+		}
+
+		o := offer.Order(payer)
+		o.DataSelector = sel
+
+		subscribeEvents, err := fapi.ClientGetRetrievalUpdates(ctx)
+		if err != nil {
+			return nil, xerrors.Errorf("error setting up retrieval updates: %w", err)
+		}
+		retrievalRes, err := fapi.ClientRetrieve(ctx, o)
+		if err != nil {
+			return nil, xerrors.Errorf("error setting up retrieval: %w", err)
+		}
+
+		start := time.Now()
+	readEvents:
+		for {
+			var evt lapi.RetrievalInfo
+			select {
+			case <-ctx.Done():
+				return nil, xerrors.New("Retrieval Timed Out")
+			case evt = <-subscribeEvents:
+				if evt.ID != retrievalRes.DealID {
+					// we can't check the deal ID ahead of time because:
+					// 1. We need to subscribe before retrieving.
+					// 2. We won't know the deal ID until after retrieving.
+					continue
+				}
+			}
+
+			event := "New"
+			if evt.Event != nil {
+				event = retrievalmarket.ClientEvents[*evt.Event]
+			}
+
+			printf("Recv %s, Paid %s, %s (%s), %s\n",
+				types.SizeStr(types.NewInt(evt.BytesReceived)),
+				types.FIL(evt.TotalPaid),
+				strings.TrimPrefix(event, "ClientEvent"),
+				strings.TrimPrefix(retrievalmarket.DealStatuses[evt.Status], "DealStatus"),
+				time.Now().Sub(start).Truncate(time.Millisecond),
+			)
+
+			switch evt.Status {
+			case retrievalmarket.DealStatusCompleted:
+				break readEvents
+			case retrievalmarket.DealStatusRejected:
+				return nil, xerrors.Errorf("Retrieval Proposal Rejected: %s", evt.Message)
+			case
+				retrievalmarket.DealStatusDealNotFound,
+				retrievalmarket.DealStatusErrored:
+				return nil, xerrors.Errorf("Retrieval Error: %s", evt.Message)
+			}
+		}
+
+		eref = &lapi.ExportRef{
+			Root:   file,
+			DealID: retrievalRes.DealID,
+		}
+	}
+
+	return eref, nil
+}
+
+var retrFlagsCommon = []cli.Flag{
+	&cli.StringFlag{
+		Name:  "from",
+		Usage: "address to send transactions from",
+	},
+	&cli.StringFlag{
+		Name:    "provider",
+		Usage:   "provider to use for retrieval, if not present it'll use local discovery",
+		Aliases: []string{"miner"},
+	},
+	&cli.StringFlag{
+		Name:  "maxPrice",
+		Usage: fmt.Sprintf("maximum price the client is willing to consider (default: %s FIL)", DefaultMaxRetrievePrice),
+	},
+	&cli.StringFlag{
+		Name:  "pieceCid",
+		Usage: "require data to be retrieved from a specific Piece CID",
+	},
+	&cli.BoolFlag{
+		Name: "allow-local",
+		// todo: default to true?
+	},
+}
+
+var clientRetrieveCmd = &cli.Command{
+	Name:      "retrieve",
+	Usage:     "Retrieve data from network",
+	ArgsUsage: "[dataCid outputPath]",
+	Description: `Retrieve data from the Filecoin network.
+
+The retrieve command will attempt to find a provider make a retrieval deal with
+them. In case a provider can't be found, it can be specified with the --provider
+flag.
+
+By default the data will be interpreted as DAG-PB UnixFSv1 File. Alternatively
+a CAR file containing the raw IPLD graph can be exported by setting the --car
+flag.
+
+Partial Retrieval:
+
+The --data-selector flag can be used to specify a sub-graph to fetch. The
+selector can be specified as either IPLD datamodel text-path selector, or IPLD
+json selector.
+
+In case of unixfs retrieval, the selector must point at a single root node, and
+match the entire graph under that node.
+
+In case of CAR retrieval, the selector must have one common "sub-root" node.
+
+Examples:
+
+- Retrieve a file by CID
+	$ lotus client retrieve Qm... my-file.txt
+
+- Retrieve a file by CID from f0123
+	$ lotus client retrieve --provider f0123 Qm... my-file.txt
+
+- Retrieve a first file from a specified directory
+	$ lotus client retrieve --data-selector /Links/0/Hash Qm... my-file.txt
+`,
+	Flags: append([]cli.Flag{
+		&cli.BoolFlag{
+			Name:  "car",
+			Usage: "Export to a car file instead of a regular file",
+		},
+		&cli.StringFlag{
+			Name:    "data-selector",
+			Aliases: []string{"datamodel-path-selector"},
+			Usage:   "IPLD datamodel text-path selector, or IPLD json selector",
+		},
+		&cli.BoolFlag{
+			Name:  "car-export-merkle-proof",
+			Usage: "(requires --data-selector and --car) Export data-selector merkle proof",
+		},
+	}, retrFlagsCommon...),
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 2 {
+			return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments"))
+		}
+
+		if cctx.Bool("car-export-merkle-proof") {
+			if !cctx.Bool("car") || !cctx.IsSet("data-selector") {
+				return ShowHelp(cctx, fmt.Errorf("--car-export-merkle-proof requires --car and --data-selector"))
+			}
+		}
+
+		fapi, closer, err := GetFullNodeAPIV1(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		afmt := NewAppFmt(cctx.App)
+
+		var s *lapi.Selector
+		if sel := lapi.Selector(cctx.String("data-selector")); sel != "" {
+			s = &sel
+		}
+
+		eref, err := retrieve(ctx, cctx, fapi, s, afmt.Printf)
+		if err != nil {
+			return err
+		}
+
+		if s != nil {
+			eref.DAGs = append(eref.DAGs, lapi.DagSpec{DataSelector: s, ExportMerkleProof: cctx.Bool("car-export-merkle-proof")})
+		}
+
+		err = fapi.ClientExport(ctx, *eref, lapi.FileRef{
+			Path:  cctx.Args().Get(1),
+			IsCAR: cctx.Bool("car"),
+		})
+		if err != nil {
+			return err
+		}
+		afmt.Println("Success")
+		return nil
+	},
+}
+
+func ClientExportStream(apiAddr string, apiAuth http.Header, eref lapi.ExportRef, car bool) (io.ReadCloser, error) {
+	rj, err := json.Marshal(eref)
+	if err != nil {
+		return nil, xerrors.Errorf("marshaling export ref: %w", err)
+	}
+
+	ma, err := multiaddr.NewMultiaddr(apiAddr)
+	if err == nil {
+		_, addr, err := manet.DialArgs(ma)
+		if err != nil {
+			return nil, err
+		}
+
+		// todo: make cliutil helpers for this
+		apiAddr = "http://" + addr
+	}
+
+	aa, err := url.Parse(apiAddr)
+	if err != nil {
+		return nil, xerrors.Errorf("parsing api address: %w", err)
+	}
+	switch aa.Scheme {
+	case "ws":
+		aa.Scheme = "http"
+	case "wss":
+		aa.Scheme = "https"
+	}
+
+	aa.Path = path.Join(aa.Path, "rest/v0/export")
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s?car=%t&export=%s", aa, car, url.QueryEscape(string(rj))), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header = apiAuth
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		em, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, xerrors.Errorf("reading error body: %w", err)
+		}
+
+		resp.Body.Close() // nolint
+		return nil, xerrors.Errorf("getting root car: http %d: %s", resp.StatusCode, string(em))
+	}
+
+	return resp.Body, nil
+}
+
+var clientRetrieveCatCmd = &cli.Command{
+	Name:      "cat",
+	Usage:     "Show data from network",
+	ArgsUsage: "[dataCid]",
+	Flags: append([]cli.Flag{
+		&cli.BoolFlag{
+			Name:  "ipld",
+			Usage: "list IPLD datamodel links",
+		},
+		&cli.StringFlag{
+			Name:  "data-selector",
+			Usage: "IPLD datamodel text-path selector, or IPLD json selector",
+		},
+	}, retrFlagsCommon...),
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 1 {
+			return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments"))
+		}
+
+		ainfo, err := GetAPIInfo(cctx, repo.FullNode)
+		if err != nil {
+			return xerrors.Errorf("could not get API info: %w", err)
+		}
+
+		fapi, closer, err := GetFullNodeAPIV1(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		afmt := NewAppFmt(cctx.App)
+
+		sel := lapi.Selector(cctx.String("data-selector"))
+		selp := &sel
+		if sel == "" {
+			selp = nil
+		}
+
+		eref, err := retrieve(ctx, cctx, fapi, selp, afmt.Printf)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println() // separate retrieval events from results
+
+		if sel != "" {
+			eref.DAGs = append(eref.DAGs, lapi.DagSpec{DataSelector: &sel})
+		}
+
+		rc, err := ClientExportStream(ainfo.Addr, ainfo.AuthHeader(), *eref, false)
+		if err != nil {
+			return err
+		}
+		defer rc.Close() // nolint
+
+		_, err = io.Copy(os.Stdout, rc)
+		return err
+	},
+}
+
+func pathToSel(psel string, matchTraversal bool, sub builder.SelectorSpec) (lapi.Selector, error) {
+	rs, err := textselector.SelectorSpecFromPath(textselector.Expression(psel), matchTraversal, sub)
+	if err != nil {
+		return "", xerrors.Errorf("failed to parse path-selector: %w", err)
+	}
+
+	var b bytes.Buffer
+	if err := dagjson.Encode(rs.Node(), &b); err != nil {
+		return "", err
+	}
+
+	return lapi.Selector(b.String()), nil
+}
+
+var clientRetrieveLsCmd = &cli.Command{
+	Name:      "ls",
+	Usage:     "List object links",
+	ArgsUsage: "[dataCid]",
+	Flags: append([]cli.Flag{
+		&cli.BoolFlag{
+			Name:  "ipld",
+			Usage: "list IPLD datamodel links",
+		},
+		&cli.IntFlag{
+			Name:  "depth",
+			Usage: "list links recursively up to the specified depth",
+			Value: 1,
+		},
+		&cli.StringFlag{
+			Name:  "data-selector",
+			Usage: "IPLD datamodel text-path selector, or IPLD json selector",
+		},
+	}, retrFlagsCommon...),
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 1 {
+			return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments"))
+		}
+
+		ainfo, err := GetAPIInfo(cctx, repo.FullNode)
+		if err != nil {
+			return xerrors.Errorf("could not get API info: %w", err)
+		}
+
+		fapi, closer, err := GetFullNodeAPIV1(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		afmt := NewAppFmt(cctx.App)
+
+		dataSelector := lapi.Selector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth")))
+
+		if cctx.IsSet("data-selector") {
+			ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
+			dataSelector, err = pathToSel(cctx.String("data-selector"), cctx.Bool("ipld"),
+				ssb.ExploreUnion(
+					ssb.Matcher(),
+					ssb.ExploreAll(
+						ssb.ExploreRecursive(selector.RecursionLimitDepth(int64(cctx.Int("depth"))), ssb.ExploreAll(ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursiveEdge()))),
+					)))
+			if err != nil {
+				return xerrors.Errorf("parsing datamodel path: %w", err)
+			}
+		}
+
+		eref, err := retrieve(ctx, cctx, fapi, &dataSelector, afmt.Printf)
+		if err != nil {
+			return xerrors.Errorf("retrieve: %w", err)
+		}
+
+		fmt.Println() // separate retrieval events from results
+
+		eref.DAGs = append(eref.DAGs, lapi.DagSpec{
+			DataSelector: &dataSelector,
+		})
+
+		rc, err := ClientExportStream(ainfo.Addr, ainfo.AuthHeader(), *eref, true)
+		if err != nil {
+			return xerrors.Errorf("export: %w", err)
+		}
+		defer rc.Close() // nolint
+
+		var memcar bytes.Buffer
+		_, err = io.Copy(&memcar, rc)
+		if err != nil {
+			return err
+		}
+
+		cbs, err := blockstore.NewReadOnly(&bytesReaderAt{bytes.NewReader(memcar.Bytes())}, nil,
+			carv2.ZeroLengthSectionAsEOF(true),
+			blockstore.UseWholeCIDs(true))
+		if err != nil {
+			return xerrors.Errorf("opening car blockstore: %w", err)
+		}
+
+		roots, err := cbs.Roots()
+		if err != nil {
+			return xerrors.Errorf("getting roots: %w", err)
+		}
+
+		if len(roots) != 1 {
+			return xerrors.Errorf("expected 1 car root, got %d", len(roots))
+		}
+		dserv := merkledag.NewDAGService(blockservice.New(cbs, offline.Exchange(cbs)))
+
+		if !cctx.Bool("ipld") {
+			links, err := dserv.GetLinks(ctx, roots[0])
+			if err != nil {
+				return xerrors.Errorf("getting links: %w", err)
+			}
+
+			for _, link := range links {
+				fmt.Printf("%s %s\t%d\n", link.Cid, link.Name, link.Size)
+			}
+		} else {
+			jsel := lapi.Selector(fmt.Sprintf(`{"R":{"l":{"depth":%d},":>":{"a":{">":{"|":[{"@":{}},{".":{}}]}}}}}`, cctx.Int("depth")))
+
+			if cctx.IsSet("data-selector") {
+				ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
+				jsel, err = pathToSel(cctx.String("data-selector"), false,
+					ssb.ExploreRecursive(selector.RecursionLimitDepth(int64(cctx.Int("depth"))), ssb.ExploreAll(ssb.ExploreUnion(ssb.Matcher(), ssb.ExploreRecursiveEdge()))),
+				)
+			}
+
+			sel, _ := selectorparse.ParseJSONSelector(string(jsel))
+
+			if err := utils.TraverseDag(
+				ctx,
+				dserv,
+				roots[0],
+				sel,
+				func(p traversal.Progress, n ipld.Node, r traversal.VisitReason) error {
+					if r == traversal.VisitReason_SelectionMatch {
+						fmt.Println(p.Path)
+					}
+					return nil
+				},
+			); err != nil {
+				return err
+			}
+		}
+
+		return err
+	},
+}
+
+type bytesReaderAt struct {
+	btr *bytes.Reader
+}
+
+func (b bytesReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
+	return b.btr.ReadAt(p, off)
+}
+
+var _ io.ReaderAt = &bytesReaderAt{}
@@ -220,8 +220,8 @@ var filplusCheckClientCmd = &cli.Command{
 }

 var filplusCheckNotaryCmd = &cli.Command{
-	Name:  "check-notaries-datacap",
-	Usage: "check notaries remaining bytes",
+	Name:  "check-notary-datacap",
+	Usage: "check a notary's remaining bytes",
 	Action: func(cctx *cli.Context) error {
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must specify notary address to check")
@@ -51,6 +51,7 @@ var multisigCmd = &cli.Command{
 		msigProposeCmd,
 		msigRemoveProposeCmd,
 		msigApproveCmd,
+		msigCancelCmd,
 		msigAddProposeCmd,
 		msigAddApproveCmd,
 		msigAddCancelCmd,
@@ -159,6 +160,8 @@ var msigCreateCmd = &cli.Command{

 		msgCid := sm.Cid()

+		fmt.Println("sent create in message: ", msgCid)
+
 		// wait for it to get mined into a block
 		wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence")), build.Finality, true)
 		if err != nil {
@@ -448,7 +451,7 @@ var msigProposeCmd = &cli.Command{

 		msgCid := sm.Cid()

-		fmt.Println("send proposal in message: ", msgCid)
+		fmt.Println("sent proposal in message: ", msgCid)

 		wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence")), build.Finality, true)
 		if err != nil {
@@ -612,6 +615,131 @@ var msigApproveCmd = &cli.Command{
 	},
 }

+var msigCancelCmd = &cli.Command{
+	Name:      "cancel",
+	Usage:     "Cancel a multisig message",
+	ArgsUsage: "<multisigAddress messageId> [destination value [methodId methodParams]]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "from",
+			Usage: "account to send the cancel message from",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() < 2 {
+			return ShowHelp(cctx, fmt.Errorf("must pass at least multisig address and message ID"))
+		}
+
+		if cctx.Args().Len() > 2 && cctx.Args().Len() < 4 {
+			return ShowHelp(cctx, fmt.Errorf("usage: msig cancel <msig addr> <message ID> <desination> <value>"))
+		}
+
+		if cctx.Args().Len() > 4 && cctx.Args().Len() != 6 {
+			return ShowHelp(cctx, fmt.Errorf("usage: msig cancel <msig addr> <message ID> <desination> <value> [ <method> <params> ]"))
+		}
+
+		srv, err := GetFullNodeServices(cctx)
+		if err != nil {
+			return err
+		}
+		defer srv.Close() //nolint:errcheck
+
+		api := srv.FullNodeAPI()
+		ctx := ReqContext(cctx)
+
+		msig, err := address.NewFromString(cctx.Args().Get(0))
+		if err != nil {
+			return err
+		}
+
+		txid, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64)
+		if err != nil {
+			return err
+		}
+
+		var from address.Address
+		if cctx.IsSet("from") {
+			f, err := address.NewFromString(cctx.String("from"))
+			if err != nil {
+				return err
+			}
+			from = f
+		} else {
+			defaddr, err := api.WalletDefaultAddress(ctx)
+			if err != nil {
+				return err
+			}
+			from = defaddr
+		}
+
+		var msgCid cid.Cid
+		if cctx.Args().Len() == 2 {
+			proto, err := api.MsigCancel(ctx, msig, txid, from)
+			if err != nil {
+				return err
+			}
+
+			sm, err := InteractiveSend(ctx, cctx, srv, proto)
+			if err != nil {
+				return err
+			}
+
+			msgCid = sm.Cid()
+		} else {
+			dest, err := address.NewFromString(cctx.Args().Get(2))
+			if err != nil {
+				return err
+			}
+
+			value, err := types.ParseFIL(cctx.Args().Get(3))
+			if err != nil {
+				return err
+			}
+
+			var method uint64
+			var params []byte
+			if cctx.Args().Len() == 6 {
+				m, err := strconv.ParseUint(cctx.Args().Get(4), 10, 64)
+				if err != nil {
+					return err
+				}
+				method = m
+
+				p, err := hex.DecodeString(cctx.Args().Get(5))
+				if err != nil {
+					return err
+				}
+				params = p
+			}
+
+			proto, err := api.MsigCancelTxnHash(ctx, msig, txid, dest, types.BigInt(value), from, method, params)
+			if err != nil {
+				return err
+			}
+
+			sm, err := InteractiveSend(ctx, cctx, srv, proto)
+			if err != nil {
+				return err
+			}
+
+			msgCid = sm.Cid()
+		}
+
+		fmt.Println("sent cancel in message: ", msgCid)
+
+		wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence")), build.Finality, true)
+		if err != nil {
+			return err
+		}
+
+		if wait.Receipt.ExitCode != 0 {
+			return fmt.Errorf("cancel returned exit %d", wait.Receipt.ExitCode)
+		}
+
+		return nil
+	},
+}
+
 var msigRemoveProposeCmd = &cli.Command{
 	Name:      "propose-remove",
 	Usage:     "Propose to remove a signer",
@@ -1490,7 +1618,7 @@ var msigLockCancelCmd = &cli.Command{
 			return actErr
 		}

-		proto, err := api.MsigCancel(ctx, msig, txid, msig, big.Zero(), from, uint64(multisig.Methods.LockBalance), params)
+		proto, err := api.MsigCancelTxnHash(ctx, msig, txid, msig, big.Zero(), from, uint64(multisig.Methods.LockBalance), params)
 		if err != nil {
 			return err
 		}
@@ -1,6 +1,7 @@
 package cli

 import (
+	"context"
 	"fmt"
 	"time"

@@ -10,8 +11,22 @@ import (
 var WaitApiCmd = &cli.Command{
 	Name:  "wait-api",
 	Usage: "Wait for lotus api to come online",
+	Flags: []cli.Flag{
+		&cli.DurationFlag{
+			Name:  "timeout",
+			Usage: "duration to wait till fail",
+			Value: time.Second * 30,
+		},
+	},
 	Action: func(cctx *cli.Context) error {
-		for i := 0; i < 30; i++ {
+		ctx := ReqContext(cctx)
+		ctx, cancel := context.WithTimeout(ctx, cctx.Duration("timeout"))
+		defer cancel()
+		for {
+			if ctx.Err() != nil {
+				break
+			}
+
 			api, closer, err := GetAPI(cctx)
 			if err != nil {
 				fmt.Printf("Not online yet... (%s)\n", err)
@@ -20,8 +35,6 @@ var WaitApiCmd = &cli.Command{
 			}
 			defer closer()

-			ctx := ReqContext(cctx)
-
 			_, err = api.Version(ctx)
 			if err != nil {
 				return err
@@ -29,6 +42,11 @@ var WaitApiCmd = &cli.Command{

 			return nil
 		}
-		return fmt.Errorf("timed out waiting for api to come online")
+
+		if ctx.Err() == context.DeadlineExceeded {
+			return fmt.Errorf("timed out waiting for api to come online")
+		}
+
+		return ctx.Err()
 	},
 }
@@ -647,9 +647,9 @@ var walletMarketWithdraw = &cli.Command{
 				return err
 			}

-			fmt.Printf("Successfully withdrew %s FIL\n", withdrawn)
+			fmt.Printf("Successfully withdrew %s \n", types.FIL(withdrawn))
 			if withdrawn.LessThan(amt) {
-				fmt.Printf("Note that this is less than the requested amount of %s FIL\n", amt)
+				fmt.Printf("Note that this is less than the requested amount of %s \n", types.FIL(amt))
 			}
 		}

@@ -128,7 +128,7 @@ var watchHeadCmd = &cli.Command{
 			return
 		}()

-		restart, err := notifyHandler(name, nCh, sCh)
+		restart, err := notifyHandler(ctx, name, nCh, sCh)
 		if err != nil {
 			return err
 		}
@@ -1,21 +1,22 @@
 package main

 import (
+	"context"
 	"os"

 	"github.com/coreos/go-systemd/v22/dbus"
 )

-func notifyHandler(n string, ch chan interface{}, sCh chan os.Signal) (string, error) {
+func notifyHandler(ctx context.Context, n string, ch chan interface{}, sCh chan os.Signal) (string, error) {
 	select {
 	// alerts to restart systemd unit
 	case <-ch:
 		statusCh := make(chan string, 1)
-		c, err := dbus.New()
+		c, err := dbus.NewWithContext(ctx)
 		if err != nil {
 			return "", err
 		}
-		_, err = c.TryRestartUnit(n, "fail", statusCh)
+		_, err = c.TryRestartUnitContext(ctx, n, "fail", statusCh)
 		if err != nil {
 			return "", err
 		}
@@ -257,7 +257,7 @@ var actorWithdrawCmd = &cli.Command{
 			amount = abi.TokenAmount(f)

 			if amount.GreaterThan(available) {
-				return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", amount, available)
+				return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", types.FIL(amount), types.FIL(available))
 			}
 		}

@@ -283,6 +283,7 @@ var actorWithdrawCmd = &cli.Command{

 		// wait for it to get mined into a block
 		fmt.Printf("waiting for %d epochs for confirmation..\n", uint64(cctx.Int("confidence")))
+
 		wait, err := api.StateWaitMsg(ctx, smsg.Cid(), uint64(cctx.Int("confidence")))
 		if err != nil {
 			return err
@@ -305,9 +306,9 @@ var actorWithdrawCmd = &cli.Command{
 				return err
 			}

-			fmt.Printf("Successfully withdrew %s FIL\n", withdrawn)
+			fmt.Printf("Successfully withdrew %s \n", types.FIL(withdrawn))
 			if withdrawn.LessThan(amount) {
-				fmt.Printf("Note that this is less than the requested amount of %s FIL\n", amount)
+				fmt.Printf("Note that this is less than the requested amount of %s\n", types.FIL(amount))
 			}
 		}

@@ -13,6 +13,8 @@ import (
 	"path/filepath"
 	"strconv"

+	power6 "github.com/filecoin-project/specs-actors/v6/actors/builtin/power"
+
 	"github.com/docker/go-units"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-datastore"
@@ -644,11 +646,26 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, peerid peer.ID,
 		return address.Address{}, err
 	}

+	sender := owner
+	if fromstr := cctx.String("from"); fromstr != "" {
+		faddr, err := address.NewFromString(fromstr)
+		if err != nil {
+			return address.Undef, fmt.Errorf("could not parse from address: %w", err)
+		}
+		sender = faddr
+	}
+
+	// make sure the sender account exists on chain
+	_, err = api.StateLookupID(ctx, owner, types.EmptyTSK)
+	if err != nil {
+		return address.Undef, xerrors.Errorf("sender must exist on chain: %w", err)
+	}
+
 	// make sure the worker account exists on chain
 	_, err = api.StateLookupID(ctx, worker, types.EmptyTSK)
 	if err != nil {
 		signed, err := api.MpoolPushMessage(ctx, &types.Message{
-			From:  owner,
+			From:  sender,
 			To:    worker,
 			Value: types.NewInt(0),
 		}, nil)
@@ -668,35 +685,46 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, peerid peer.ID,
 		}
 	}

-	nv, err := api.StateNetworkVersion(ctx, types.EmptyTSK)
+	// make sure the owner account exists on chain
+	_, err = api.StateLookupID(ctx, owner, types.EmptyTSK)
 	if err != nil {
-		return address.Undef, xerrors.Errorf("getting network version: %w", err)
+		signed, err := api.MpoolPushMessage(ctx, &types.Message{
+			From:  sender,
+			To:    owner,
+			Value: types.NewInt(0),
+		}, nil)
+		if err != nil {
+			return address.Undef, xerrors.Errorf("push owner init: %w", err)
+		}
+
+		log.Infof("Initializing owner account %s, message: %s", worker, signed.Cid())
+		log.Infof("Waiting for confirmation")
+
+		mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true)
+		if err != nil {
+			return address.Undef, xerrors.Errorf("waiting for owner init: %w", err)
+		}
+		if mw.Receipt.ExitCode != 0 {
+			return address.Undef, xerrors.Errorf("initializing owner account failed: exit code %d", mw.Receipt.ExitCode)
+		}
 	}

-	spt, err := miner.SealProofTypeFromSectorSize(abi.SectorSize(ssize), nv)
+	// Note: the correct thing to do would be to call SealProofTypeFromSectorSize if actors version is v3 or later, but this still works
+	spt, err := miner.WindowPoStProofTypeFromSectorSize(abi.SectorSize(ssize))
 	if err != nil {
-		return address.Undef, xerrors.Errorf("getting seal proof type: %w", err)
+		return address.Undef, xerrors.Errorf("getting post proof type: %w", err)
 	}

-	params, err := actors.SerializeParams(&power2.CreateMinerParams{
-		Owner:         owner,
-		Worker:        worker,
-		SealProofType: spt,
-		Peer:          abi.PeerID(peerid),
+	params, err := actors.SerializeParams(&power6.CreateMinerParams{
+		Owner:               owner,
+		Worker:              worker,
+		WindowPoStProofType: spt,
+		Peer:                abi.PeerID(peerid),
 	})
 	if err != nil {
 		return address.Undef, err
 	}

-	sender := owner
-	if fromstr := cctx.String("from"); fromstr != "" {
-		faddr, err := address.NewFromString(fromstr)
-		if err != nil {
-			return address.Undef, fmt.Errorf("could not parse from address: %w", err)
-		}
-		sender = faddr
-	}
-
 	createStorageMinerMsg := &types.Message{
 		To:    power.Address,
 		From:  sender,
@@ -352,6 +352,7 @@ var storageDealsCmd = &cli.Command{
 		resetBlocklistCmd,
 		setSealDurationCmd,
 		dealsPendingPublish,
+		dealsRetryPublish,
 	},
 }

@@ -910,6 +911,36 @@ var dealsPendingPublish = &cli.Command{
 	},
 }

+var dealsRetryPublish = &cli.Command{
+	Name:      "retry-publish",
+	Usage:     "retry publishing a deal",
+	ArgsUsage: "<proposal CID>",
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return cli.ShowCommandHelp(cctx, cctx.Command.Name)
+		}
+		api, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		propcid := cctx.Args().First()
+		fmt.Printf("retrying deal with proposal-cid: %s\n", propcid)
+
+		cid, err := cid.Decode(propcid)
+		if err != nil {
+			return err
+		}
+		if err := api.MarketRetryPublishDeal(ctx, cid); err != nil {
+			return xerrors.Errorf("retrying publishing deal: %w", err)
+		}
+		fmt.Println("retried to publish deal")
+		return nil
+	},
+}
+
 func listDealsWithJSON(cctx *cli.Context) error {
 	node, closer, err := lcli.GetMarketsAPI(cctx)
 	if err != nil {
@@ -6,6 +6,7 @@ import (
 	"text/tabwriter"

 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/tablewriter"
 	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 )
@@ -48,6 +49,12 @@ var piecesListPiecesCmd = &cli.Command{
 var piecesListCidInfosCmd = &cli.Command{
 	Name:  "list-cids",
 	Usage: "list registered payload CIDs",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:    "verbose",
+			Aliases: []string{"v"},
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		nodeApi, closer, err := lcli.GetMarketsAPI(cctx)
 		if err != nil {
@@ -61,9 +68,54 @@ var piecesListCidInfosCmd = &cli.Command{
 			return err
 		}

+		w := tablewriter.New(tablewriter.Col("CID"),
+			tablewriter.Col("Piece"),
+			tablewriter.Col("BlockOffset"),
+			tablewriter.Col("BlockLen"),
+			tablewriter.Col("Deal"),
+			tablewriter.Col("Sector"),
+			tablewriter.Col("DealOffset"),
+			tablewriter.Col("DealLen"),
+		)
+
 		for _, c := range cids {
-			fmt.Println(c)
+			if !cctx.Bool("verbose") {
+				fmt.Println(c)
+				continue
+			}
+
+			ci, err := nodeApi.PiecesGetCIDInfo(ctx, c)
+			if err != nil {
+				fmt.Printf("Error getting CID info: %s\n", err)
+				continue
+			}
+
+			for _, location := range ci.PieceBlockLocations {
+				pi, err := nodeApi.PiecesGetPieceInfo(ctx, location.PieceCID)
+				if err != nil {
+					fmt.Printf("Error getting piece info: %s\n", err)
+					continue
+				}
+
+				for _, deal := range pi.Deals {
+					w.Write(map[string]interface{}{
+						"CID":         c,
+						"Piece":       location.PieceCID,
+						"BlockOffset": location.RelOffset,
+						"BlockLen":    location.BlockSize,
+						"Deal":        deal.DealID,
+						"Sector":      deal.SectorID,
+						"DealOffset":  deal.Offset,
+						"DealLen":     deal.Length,
+					})
+				}
+			}
 		}
+
+		if cctx.Bool("verbose") {
+			return w.Flush(os.Stdout)
+		}
+
 		return nil
 	},
 }
@@ -17,6 +17,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/specs-storage/storage"
 )

@@ -360,6 +361,10 @@ var provingCheckProvableCmd = &cli.Command{
 			Name:  "slow",
 			Usage: "run slower checks",
 		},
+		&cli.StringFlag{
+			Name:  "storage-id",
+			Usage: "filter sectors by storage path (path id)",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() != 1 {
@@ -408,6 +413,21 @@ var provingCheckProvableCmd = &cli.Command{
 		tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
 		_, _ = fmt.Fprintln(tw, "deadline\tpartition\tsector\tstatus")

+		var filter map[abi.SectorID]struct{}
+
+		if cctx.IsSet("storage-id") {
+			sl, err := sapi.StorageList(ctx)
+			if err != nil {
+				return err
+			}
+			decls := sl[stores.ID(cctx.String("storage-id"))]
+
+			filter = map[abi.SectorID]struct{}{}
+			for _, decl := range decls {
+				filter[decl.SectorID] = struct{}{}
+			}
+		}
+
 		for parIdx, par := range partitions {
 			sectors := make(map[abi.SectorNumber]struct{})

@@ -418,13 +438,21 @@ var provingCheckProvableCmd = &cli.Command{

 			var tocheck []storage.SectorRef
 			for _, info := range sectorInfos {
+				si := abi.SectorID{
+					Miner:  abi.ActorID(mid),
+					Number: info.SectorNumber,
+				}
+
+				if filter != nil {
+					if _, found := filter[si]; !found {
+						continue
+					}
+				}
+
 				sectors[info.SectorNumber] = struct{}{}
 				tocheck = append(tocheck, storage.SectorRef{
 					ProofType: info.SealProof,
-					ID: abi.SectorID{
-						Miner:  abi.ActorID(mid),
-						Number: info.SectorNumber,
-					},
+					ID:        si,
 				})
 			}

@@ -4,6 +4,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"math"
 	"os"
 	"sort"
 	"strings"
@@ -32,6 +33,17 @@ var sealingCmd = &cli.Command{
 	},
 }

+var barCols = float64(64)
+
+func barString(total, y, g float64) string {
+	yBars := int(math.Round(y / total * barCols))
+	gBars := int(math.Round(g / total * barCols))
+	eBars := int(barCols) - yBars - gBars
+	return color.YellowString(strings.Repeat("|", yBars)) +
+		color.GreenString(strings.Repeat("|", gBars)) +
+		strings.Repeat(" ", eBars)
+}
+
 var sealingWorkersCmd = &cli.Command{
 	Name:  "workers",
 	Usage: "list workers",
@@ -77,7 +89,7 @@ var sealingWorkersCmd = &cli.Command{
 		for _, stat := range st {
 			gpuUse := "not "
 			gpuCol := color.FgBlue
-			if stat.GpuUsed {
+			if stat.GpuUsed > 0 {
 				gpuCol = color.FgGreen
 				gpuUse = ""
 			}
@@ -89,56 +101,43 @@ var sealingWorkersCmd = &cli.Command{

 			fmt.Printf("Worker %s, host %s%s\n", stat.id, color.MagentaString(stat.Info.Hostname), disabled)

-			var barCols = uint64(64)
-			cpuBars := int(stat.CpuUse * barCols / stat.Info.Resources.CPUs)
-			cpuBar := strings.Repeat("|", cpuBars)
-			if int(barCols)-cpuBars >= 0 {
-				cpuBar += strings.Repeat(" ", int(barCols)-cpuBars)
-			}
-
 			fmt.Printf("\tCPU:  [%s] %d/%d core(s) in use\n",
-				color.GreenString(cpuBar), stat.CpuUse, stat.Info.Resources.CPUs)
+				barString(float64(stat.Info.Resources.CPUs), 0, float64(stat.CpuUse)), stat.CpuUse, stat.Info.Resources.CPUs)

-			ramBarsRes := int(stat.Info.Resources.MemReserved * barCols / stat.Info.Resources.MemPhysical)
-			ramBarsUsed := int(stat.MemUsedMin * barCols / stat.Info.Resources.MemPhysical)
-			ramRepeatSpace := int(barCols) - (ramBarsUsed + ramBarsRes)
-
-			colorFunc := color.YellowString
-			if ramRepeatSpace < 0 {
-				ramRepeatSpace = 0
-				colorFunc = color.RedString
+			ramTotal := stat.Info.Resources.MemPhysical
+			ramTasks := stat.MemUsedMin
+			ramUsed := stat.Info.Resources.MemUsed
+			var ramReserved uint64 = 0
+			if ramUsed > ramTasks {
+				ramReserved = ramUsed - ramTasks
 			}
-
-			ramBar := colorFunc(strings.Repeat("|", ramBarsRes)) +
-				color.GreenString(strings.Repeat("|", ramBarsUsed)) +
-				strings.Repeat(" ", ramRepeatSpace)
-
-			vmem := stat.Info.Resources.MemPhysical + stat.Info.Resources.MemSwap
-
-			vmemBarsRes := int(stat.Info.Resources.MemReserved * barCols / vmem)
-			vmemBarsUsed := int(stat.MemUsedMax * barCols / vmem)
-			vmemRepeatSpace := int(barCols) - (vmemBarsUsed + vmemBarsRes)
-
-			colorFunc = color.YellowString
-			if vmemRepeatSpace < 0 {
-				vmemRepeatSpace = 0
-				colorFunc = color.RedString
-			}
-
-			vmemBar := colorFunc(strings.Repeat("|", vmemBarsRes)) +
-				color.GreenString(strings.Repeat("|", vmemBarsUsed)) +
-				strings.Repeat(" ", vmemRepeatSpace)
+			ramBar := barString(float64(ramTotal), float64(ramReserved), float64(ramTasks))

 			fmt.Printf("\tRAM:  [%s] %d%% %s/%s\n", ramBar,
-				(stat.Info.Resources.MemReserved+stat.MemUsedMin)*100/stat.Info.Resources.MemPhysical,
-				types.SizeStr(types.NewInt(stat.Info.Resources.MemReserved+stat.MemUsedMin)),
+				(ramTasks+ramReserved)*100/stat.Info.Resources.MemPhysical,
+				types.SizeStr(types.NewInt(ramTasks+ramUsed)),
 				types.SizeStr(types.NewInt(stat.Info.Resources.MemPhysical)))

-			fmt.Printf("\tVMEM: [%s] %d%% %s/%s\n", vmemBar,
-				(stat.Info.Resources.MemReserved+stat.MemUsedMax)*100/vmem,
-				types.SizeStr(types.NewInt(stat.Info.Resources.MemReserved+stat.MemUsedMax)),
-				types.SizeStr(types.NewInt(vmem)))
+			vmemTotal := stat.Info.Resources.MemPhysical + stat.Info.Resources.MemSwap
+			vmemTasks := stat.MemUsedMax
+			vmemUsed := stat.Info.Resources.MemUsed + stat.Info.Resources.MemSwapUsed
+			var vmemReserved uint64 = 0
+			if vmemUsed > vmemTasks {
+				vmemReserved = vmemUsed - vmemTasks
+			}
+			vmemBar := barString(float64(vmemTotal), float64(vmemReserved), float64(vmemTasks))

+			fmt.Printf("\tVMEM: [%s] %d%% %s/%s\n", vmemBar,
+				(vmemTasks+vmemReserved)*100/vmemTotal,
+				types.SizeStr(types.NewInt(vmemTasks+vmemReserved)),
+				types.SizeStr(types.NewInt(vmemTotal)))
+
+			if len(stat.Info.Resources.GPUs) > 0 {
+				gpuBar := barString(float64(len(stat.Info.Resources.GPUs)), 0, stat.GpuUsed)
+				fmt.Printf("\tGPU:  [%s] %.f%% %.2f/%d gpu(s) in use\n", color.GreenString(gpuBar),
+					stat.GpuUsed*100/float64(len(stat.Info.Resources.GPUs)),
+					stat.GpuUsed, len(stat.Info.Resources.GPUs))
+			}
 			for _, gpu := range stat.Info.Resources.GPUs {
 				fmt.Printf("\tGPU: %s\n", color.New(gpuCol).Sprintf("%s, %sused", gpu, gpuUse))
 			}
@@ -224,8 +223,10 @@ var sealingJobsCmd = &cli.Command{
 		for _, l := range lines {
 			state := "running"
 			switch {
-			case l.RunWait > 0:
+			case l.RunWait > 1:
 				state = fmt.Sprintf("assigned(%d)", l.RunWait-1)
+			case l.RunWait == storiface.RWPrepared:
+				state = "prepared"
 			case l.RunWait == storiface.RWRetDone:
 				if !cctx.Bool("show-ret-done") {
 					continue
@@ -95,6 +95,14 @@ over time
 			Name:  "max-storage",
 			Usage: "(for init) limit storage space for sectors (expensive for very large paths!)",
 		},
+		&cli.StringSliceFlag{
+			Name:  "groups",
+			Usage: "path group names",
+		},
+		&cli.StringSliceFlag{
+			Name:  "allow-to",
+			Usage: "path groups allowed to pull data from this path (allow all if not specified)",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
@@ -142,6 +150,8 @@ over time
 				CanSeal:    cctx.Bool("seal"),
 				CanStore:   cctx.Bool("store"),
 				MaxStorage: uint64(maxStor),
+				Groups:     cctx.StringSlice("groups"),
+				AllowTo:    cctx.StringSlice("allow-to"),
 			}

 			if !(cfg.CanStore || cfg.CanSeal) {
@@ -322,10 +332,17 @@ var storageListCmd = &cli.Command{
 				if si.CanStore {
 					fmt.Print(color.CyanString("Store"))
 				}
-				fmt.Println("")
 			} else {
 				fmt.Print(color.HiYellowString("Use: ReadOnly"))
 			}
+			fmt.Println()
+
+			if len(si.Groups) > 0 {
+				fmt.Printf("\tGroups: %s\n", strings.Join(si.Groups, ", "))
+			}
+			if len(si.AllowTo) > 0 {
+				fmt.Printf("\tAllowTo: %s\n", strings.Join(si.AllowTo, ", "))
+			}

 			if localPath, ok := local[s.ID]; ok {
 				fmt.Printf("\tLocal: %s\n", color.GreenString(localPath))
@@ -17,6 +17,7 @@ import (
 	"time"

 	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	lcli "github.com/filecoin-project/lotus/cli"

 	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"

@@ -41,7 +42,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/tools/stats"
+	"github.com/filecoin-project/lotus/tools/stats/sync"
 )

 var log = logging.Logger("main")
@@ -160,15 +161,15 @@ var findMinersCmd = &cli.Command{
 	},
 	Action: func(cctx *cli.Context) error {
 		ctx := context.Background()
-		api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path"))
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
-			log.Fatal(err)
+			return err
 		}
 		defer closer()

 		if !cctx.Bool("no-sync") {
-			if err := stats.WaitForSyncComplete(ctx, api); err != nil {
-				log.Fatal(err)
+			if err := sync.SyncWait(ctx, api); err != nil {
+				return err
 			}
 		}

@@ -245,7 +246,7 @@ var recoverMinersCmd = &cli.Command{
 	},
 	Action: func(cctx *cli.Context) error {
 		ctx := context.Background()
-		api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path"))
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -266,8 +267,8 @@ var recoverMinersCmd = &cli.Command{
 		}

 		if !cctx.Bool("no-sync") {
-			if err := stats.WaitForSyncComplete(ctx, api); err != nil {
-				log.Fatal(err)
+			if err := sync.SyncWait(ctx, api); err != nil {
+				return err
 			}
 		}

@@ -427,7 +428,7 @@ var runCmd = &cli.Command{
 		}()

 		ctx := context.Background()
-		api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path"))
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -448,12 +449,12 @@ var runCmd = &cli.Command{
 		}

 		if !cctx.Bool("no-sync") {
-			if err := stats.WaitForSyncComplete(ctx, api); err != nil {
-				log.Fatal(err)
+			if err := sync.SyncWait(ctx, api); err != nil {
+				return err
 			}
 		}

-		tipsetsCh, err := stats.GetTips(ctx, api, r.Height(), cctx.Int("head-delay"))
+		tipsetsCh, err := sync.BufferedTipsetChannel(ctx, api, r.Height(), cctx.Int("head-delay"))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -58,8 +58,11 @@ var infoCmd = &cli.Command{

 		fmt.Printf("Hostname: %s\n", info.Hostname)
 		fmt.Printf("CPUs: %d; GPUs: %v\n", info.Resources.CPUs, info.Resources.GPUs)
-		fmt.Printf("RAM: %s; Swap: %s\n", types.SizeStr(types.NewInt(info.Resources.MemPhysical)), types.SizeStr(types.NewInt(info.Resources.MemSwap)))
-		fmt.Printf("Reserved memory: %s\n", types.SizeStr(types.NewInt(info.Resources.MemReserved)))
+		fmt.Printf("RAM: %s/%s; Swap: %s/%s\n",
+			types.SizeStr(types.NewInt(info.Resources.MemUsed)),
+			types.SizeStr(types.NewInt(info.Resources.MemPhysical)),
+			types.SizeStr(types.NewInt(info.Resources.MemSwapUsed)),
+			types.SizeStr(types.NewInt(info.Resources.MemSwap)))

 		fmt.Printf("Task types: ")
 		for _, t := range ttList(tt) {
@@ -60,6 +60,7 @@ func main() {
 		storageCmd,
 		setCmd,
 		waitQuietCmd,
+		resourcesCmd,
 		tasksCmd,
 	}

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"sort"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+)
+
+var resourcesCmd = &cli.Command{
+	Name:  "resources",
+	Usage: "Manage resource table overrides",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "all",
+			Usage: "print all resource envvars",
+		},
+		&cli.BoolFlag{
+			Name:  "default",
+			Usage: "print default resource envvars",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		def := map[string]string{}
+		set := map[string]string{}
+		all := map[string]string{}
+
+		_, err := storiface.ParseResourceEnv(func(key, d string) (string, bool) {
+			if d != "" {
+				all[key] = d
+				def[key] = d
+			}
+
+			s, ok := os.LookupEnv(key)
+			if ok {
+				all[key] = s
+				set[key] = s
+			}
+
+			return s, ok
+		})
+		if err != nil {
+			return err
+		}
+
+		printMap := func(m map[string]string) {
+			var arr []string
+			for k, v := range m {
+				arr = append(arr, fmt.Sprintf("%s=%s", k, v))
+			}
+			sort.Strings(arr)
+			for _, s := range arr {
+				fmt.Println(s)
+			}
+		}
+
+		if cctx.Bool("default") {
+			printMap(def)
+		} else {
+			if cctx.Bool("all") {
+				printMap(all)
+			} else {
+				printMap(set)
+			}
+		}
+
+		return nil
+	},
+}
@@ -51,6 +51,14 @@ var storageAttachCmd = &cli.Command{
 			Name:  "max-storage",
 			Usage: "(for init) limit storage space for sectors (expensive for very large paths!)",
 		},
+		&cli.StringSliceFlag{
+			Name:  "groups",
+			Usage: "path group names",
+		},
+		&cli.StringSliceFlag{
+			Name:  "allow-to",
+			Usage: "path groups allowed to pull data from this path (allow all if not specified)",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
@@ -98,6 +106,8 @@ var storageAttachCmd = &cli.Command{
 				CanSeal:    cctx.Bool("seal"),
 				CanStore:   cctx.Bool("store"),
 				MaxStorage: uint64(maxStor),
+				Groups:     cctx.StringSlice("groups"),
+				AllowTo:    cctx.StringSlice("allow-to"),
 			}

 			if !(cfg.CanStore || cfg.CanSeal) {
@@ -104,7 +104,7 @@ var actorWithdrawCmd = &cli.Command{
 			amount = abi.TokenAmount(f)

 			if amount.GreaterThan(available) {
-				return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", amount, available)
+				return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", types.FIL(amount), types.FIL(available))
 			}
 		}

@@ -151,9 +151,9 @@ var actorWithdrawCmd = &cli.Command{
 				return err
 			}

-			fmt.Printf("Successfully withdrew %s FIL\n", withdrawn)
+			fmt.Printf("Successfully withdrew %s \n", types.FIL(withdrawn))
 			if withdrawn.LessThan(amount) {
-				fmt.Printf("Note that this is less than the requested amount of %s FIL\n", amount)
+				fmt.Printf("Note that this is less than the requested amount of %s \n", types.FIL(amount))
 			}
 		}

@@ -0,0 +1,222 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/ipfs/go-cid"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/exitcode"
+
+	lapi "github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var balancerCmd = &cli.Command{
+	Name:  "balancer",
+	Usage: "Utility for balancing tokens between multiple wallets",
+	Description: `Tokens are balanced based on the specification provided in arguments
+
+Each argument specifies an address, role, and role parameters separated by ';'
+
+Supported roles:
+	- request;[addr];[low];[high] - request tokens when balance drops to [low], topping up to [high]
+	- provide;[addr];[min] - provide tokens to other addresses as long as the balance is above [min]
+`,
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPIV1(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		type request struct {
+			addr      address.Address
+			low, high abi.TokenAmount
+		}
+		type provide struct {
+			addr address.Address
+			min  abi.TokenAmount
+		}
+
+		var requests []request
+		var provides []provide
+
+		for i, s := range cctx.Args().Slice() {
+			ss := strings.Split(s, ";")
+			switch ss[0] {
+			case "request":
+				if len(ss) != 4 {
+					return xerrors.Errorf("request role needs 4 parameters (arg %d)", i)
+				}
+
+				addr, err := address.NewFromString(ss[1])
+				if err != nil {
+					return xerrors.Errorf("parsing address in arg %d: %w", i, err)
+				}
+
+				low, err := types.ParseFIL(ss[2])
+				if err != nil {
+					return xerrors.Errorf("parsing low in arg %d: %w", i, err)
+				}
+
+				high, err := types.ParseFIL(ss[3])
+				if err != nil {
+					return xerrors.Errorf("parsing high in arg %d: %w", i, err)
+				}
+
+				if abi.TokenAmount(low).GreaterThanEqual(abi.TokenAmount(high)) {
+					return xerrors.Errorf("low must be less than high in arg %d", i)
+				}
+
+				requests = append(requests, request{
+					addr: addr,
+					low:  abi.TokenAmount(low),
+					high: abi.TokenAmount(high),
+				})
+			case "provide":
+				if len(ss) != 3 {
+					return xerrors.Errorf("provide role needs 3 parameters (arg %d)", i)
+				}
+
+				addr, err := address.NewFromString(ss[1])
+				if err != nil {
+					return xerrors.Errorf("parsing address in arg %d: %w", i, err)
+				}
+
+				min, err := types.ParseFIL(ss[2])
+				if err != nil {
+					return xerrors.Errorf("parsing min in arg %d: %w", i, err)
+				}
+
+				provides = append(provides, provide{
+					addr: addr,
+					min:  abi.TokenAmount(min),
+				})
+			default:
+				return xerrors.Errorf("unknown role '%s' in arg %d", ss[0], i)
+			}
+		}
+
+		if len(provides) == 0 {
+			return xerrors.Errorf("no provides specified")
+		}
+		if len(requests) == 0 {
+			return xerrors.Errorf("no requests specified")
+		}
+
+		const confidence = 16
+
+		var notifs <-chan []*lapi.HeadChange
+		for {
+			if notifs == nil {
+				notifs, err = api.ChainNotify(ctx)
+				if err != nil {
+					return xerrors.Errorf("chain notify error: %w", err)
+				}
+			}
+
+			var ts *types.TipSet
+		loop:
+			for {
+				time.Sleep(150 * time.Millisecond)
+				select {
+				case n := <-notifs:
+					for _, change := range n {
+						if change.Type != store.HCApply {
+							continue
+						}
+
+						ts = change.Val
+					}
+				case <-ctx.Done():
+					return nil
+				default:
+					break loop
+				}
+			}
+
+			type send struct {
+				to     address.Address
+				amt    abi.TokenAmount
+				filled bool
+			}
+			var toSend []*send
+
+			for _, req := range requests {
+				bal, err := api.StateGetActor(ctx, req.addr, ts.Key())
+				if err != nil {
+					return err
+				}
+
+				if bal.Balance.LessThan(req.low) {
+					toSend = append(toSend, &send{
+						to:  req.addr,
+						amt: big.Sub(req.high, bal.Balance),
+					})
+				}
+			}
+
+			for _, s := range toSend {
+				fmt.Printf("REQUEST %s for %s\n", types.FIL(s.amt), s.to)
+			}
+
+			var msgs []cid.Cid
+
+			for _, prov := range provides {
+				bal, err := api.StateGetActor(ctx, prov.addr, ts.Key())
+				if err != nil {
+					return err
+				}
+
+				avail := big.Sub(bal.Balance, prov.min)
+				for _, s := range toSend {
+					if s.filled {
+						continue
+					}
+					if avail.LessThan(s.amt) {
+						continue
+					}
+
+					m, err := api.MpoolPushMessage(ctx, &types.Message{
+						From:  prov.addr,
+						To:    s.to,
+						Value: s.amt,
+					}, nil)
+					if err != nil {
+						fmt.Printf("SEND ERROR %s\n", err.Error())
+					}
+					fmt.Printf("SEND %s; %s from %s TO %s\n", m.Cid(), types.FIL(s.amt), s.to, prov.addr)
+
+					msgs = append(msgs, m.Cid())
+					s.filled = true
+					avail = big.Sub(avail, s.amt)
+				}
+			}
+
+			if len(msgs) > 0 {
+				fmt.Printf("WAITING FOR %d MESSAGES\n", len(msgs))
+			}
+
+			for _, msg := range msgs {
+				ml, err := api.StateWaitMsg(ctx, msg, confidence, lapi.LookbackNoLimit, true)
+				if err != nil {
+					return err
+				}
+				if ml.Receipt.ExitCode != exitcode.Ok {
+					fmt.Printf("MSG %s NON-ZERO EXITCODE: %s\n", msg, ml.Receipt.ExitCode)
+				}
+			}
+		}
+	},
+}
@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/urfave/cli/v2"
+)
+
+var chainCmd = &cli.Command{
+	Name:  "chain",
+	Usage: "chain-related utilities",
+	Subcommands: []*cli.Command{
+		chainNullTsCmd,
+	},
+}
+
+var chainNullTsCmd = &cli.Command{
+	Name:  "latest-null",
+	Usage: "finds the most recent null tipset",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		ts, err := lcli.LoadTipSet(ctx, cctx, api)
+		if err != nil {
+			return err
+		}
+
+		for {
+			pts, err := api.ChainGetTipSet(ctx, ts.Parents())
+			if err != nil {
+				return err
+			}
+
+			if ts.Height() != pts.Height()+1 {
+				fmt.Println("null tipset at height ", ts.Height()-1)
+				return nil
+			}
+
+			ts = pts
+		}
+	},
+}
@@ -59,9 +59,12 @@ func main() {
 		signaturesCmd,
 		actorCmd,
 		minerTypesCmd,
+		minerPeeridCmd,
 		minerMultisigsCmd,
 		splitstoreCmd,
 		fr32Cmd,
+		chainCmd,
+		balancerCmd,
 	}

 	app := &cli.App{
@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/libp2p/go-libp2p-core/peer"
+
+	builtin5 "github.com/filecoin-project/specs-actors/v5/actors/builtin"
+
+	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/repo"
+	"github.com/filecoin-project/specs-actors/v4/actors/util/adt"
+	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var minerPeeridCmd = &cli.Command{
+	Name:  "miner-peerid",
+	Usage: "Scrape state to find a miner based on peerid", Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "repo",
+			Value: "~/.lotus",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := context.TODO()
+
+		if cctx.NArg() != 2 {
+			return fmt.Errorf("must pass peer id and state root")
+		}
+
+		pid, err := peer.Decode(cctx.Args().Get(0))
+		if err != nil {
+			return fmt.Errorf("failed to parse input as a peerId: %w", err)
+		}
+
+		sroot, err := cid.Decode(cctx.Args().Get(1))
+		if err != nil {
+			return fmt.Errorf("failed to parse state root: %w", err)
+		}
+
+		fsrepo, err := repo.NewFS(cctx.String("repo"))
+		if err != nil {
+			return err
+		}
+
+		lkrepo, err := fsrepo.Lock(repo.FullNode)
+		if err != nil {
+			return err
+		}
+
+		defer lkrepo.Close() //nolint:errcheck
+
+		bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
+		if err != nil {
+			return fmt.Errorf("failed to open blockstore: %w", err)
+		}
+
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
+		mds, err := lkrepo.Datastore(context.Background(), "/metadata")
+		if err != nil {
+			return err
+		}
+
+		cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
+		defer cs.Close() //nolint:errcheck
+
+		cst := cbor.NewCborStore(bs)
+		store := adt.WrapStore(ctx, cst)
+
+		tree, err := state.LoadStateTree(cst, sroot)
+		if err != nil {
+			return err
+		}
+
+		err = tree.ForEach(func(addr address.Address, act *types.Actor) error {
+			if act.Code == builtin5.StorageMinerActorCodeID {
+				ms, err := miner.Load(store, act)
+				if err != nil {
+					return err
+				}
+
+				mi, err := ms.Info()
+				if err != nil {
+					return err
+				}
+
+				if mi.PeerId != nil && *mi.PeerId == pid {
+					fmt.Println(addr)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			return xerrors.Errorf("failed to loop over actors: %w", err)
+		}
+
+		return nil
+	},
+}
@@ -2,11 +2,29 @@ package main

 import (
 	"bufio"
+	"bytes"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"

+	miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
+
+	power6 "github.com/filecoin-project/specs-actors/v6/actors/builtin/power"
+
+	"github.com/docker/go-units"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
@@ -17,6 +35,231 @@ var minerCmd = &cli.Command{
 	Usage: "miner-related utilities",
 	Subcommands: []*cli.Command{
 		minerUnpackInfoCmd,
+		minerCreateCmd,
+		minerFaultsCmd,
+	},
+}
+
+var minerFaultsCmd = &cli.Command{
+	Name:      "faults",
+	Usage:     "Display a list of faulty sectors for a SP",
+	ArgsUsage: "[minerAddress]",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "expiring-in",
+			Usage: "only list sectors that are expiring in the next <n> epochs",
+			Value: 0,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass miner address")
+		}
+
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		m, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		faultBf, err := api.StateMinerFaults(ctx, m, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		faults, err := faultBf.All(miner2.SectorsMax)
+		if err != nil {
+			return err
+		}
+
+		if len(faults) == 0 {
+			fmt.Println("no faults")
+			return nil
+		}
+
+		expEpoch := abi.ChainEpoch(cctx.Uint64("expiring-in"))
+
+		if expEpoch == 0 {
+			fmt.Print("faulty sectors: ")
+			for _, v := range faults {
+				fmt.Printf("%d ", v)
+			}
+
+			return nil
+		}
+
+		h, err := api.ChainHead(ctx)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("faulty sectors expiring in the next %d epochs: ", expEpoch)
+		for _, v := range faults {
+			ss, err := api.StateSectorExpiration(ctx, m, abi.SectorNumber(v), types.EmptyTSK)
+			if err != nil {
+				return err
+			}
+
+			if ss.Early < h.Height()+expEpoch {
+				fmt.Printf("%d ", v)
+			}
+		}
+
+		return nil
+	},
+}
+
+var minerCreateCmd = &cli.Command{
+	Name:      "create",
+	Usage:     "sends a create miner msg",
+	ArgsUsage: "[sender] [owner] [worker] [sector size]",
+	Action: func(cctx *cli.Context) error {
+		wapi, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		if cctx.Args().Len() != 4 {
+			return xerrors.Errorf("expected 4 args (sender owner worker sectorSize)")
+		}
+
+		sender, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		owner, err := address.NewFromString(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+
+		worker, err := address.NewFromString(cctx.Args().Get(2))
+		if err != nil {
+			return err
+		}
+
+		ssize, err := units.RAMInBytes(cctx.Args().Get(3))
+		if err != nil {
+			return fmt.Errorf("failed to parse sector size: %w", err)
+		}
+
+		// make sure the sender account exists on chain
+		_, err = wapi.StateLookupID(ctx, owner, types.EmptyTSK)
+		if err != nil {
+			return xerrors.Errorf("sender must exist on chain: %w", err)
+		}
+
+		// make sure the worker account exists on chain
+		_, err = wapi.StateLookupID(ctx, worker, types.EmptyTSK)
+		if err != nil {
+			signed, err := wapi.MpoolPushMessage(ctx, &types.Message{
+				From:  sender,
+				To:    worker,
+				Value: types.NewInt(0),
+			}, nil)
+			if err != nil {
+				return xerrors.Errorf("push worker init: %w", err)
+			}
+
+			log.Infof("Initializing worker account %s, message: %s", worker, signed.Cid())
+			log.Infof("Waiting for confirmation")
+
+			mw, err := wapi.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence)
+			if err != nil {
+				return xerrors.Errorf("waiting for worker init: %w", err)
+			}
+
+			if mw.Receipt.ExitCode != 0 {
+				return xerrors.Errorf("initializing worker account failed: exit code %d", mw.Receipt.ExitCode)
+			}
+		}
+
+		// make sure the owner account exists on chain
+		_, err = wapi.StateLookupID(ctx, owner, types.EmptyTSK)
+		if err != nil {
+			signed, err := wapi.MpoolPushMessage(ctx, &types.Message{
+				From:  sender,
+				To:    owner,
+				Value: types.NewInt(0),
+			}, nil)
+			if err != nil {
+				return xerrors.Errorf("push owner init: %w", err)
+			}
+
+			log.Infof("Initializing owner account %s, message: %s", worker, signed.Cid())
+			log.Infof("Wating for confirmation")
+
+			mw, err := wapi.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence)
+			if err != nil {
+				return xerrors.Errorf("waiting for owner init: %w", err)
+			}
+
+			if mw.Receipt.ExitCode != 0 {
+				return xerrors.Errorf("initializing owner account failed: exit code %d", mw.Receipt.ExitCode)
+			}
+		}
+
+		// Note: the correct thing to do would be to call SealProofTypeFromSectorSize if actors version is v3 or later, but this still works
+		spt, err := miner.WindowPoStProofTypeFromSectorSize(abi.SectorSize(ssize))
+		if err != nil {
+			return xerrors.Errorf("getting post proof type: %w", err)
+		}
+
+		params, err := actors.SerializeParams(&power6.CreateMinerParams{
+			Owner:               owner,
+			Worker:              worker,
+			WindowPoStProofType: spt,
+		})
+
+		if err != nil {
+			return err
+		}
+
+		createStorageMinerMsg := &types.Message{
+			To:    power.Address,
+			From:  sender,
+			Value: big.Zero(),
+
+			Method: power.Methods.CreateMiner,
+			Params: params,
+		}
+
+		signed, err := wapi.MpoolPushMessage(ctx, createStorageMinerMsg, nil)
+		if err != nil {
+			return xerrors.Errorf("pushing createMiner message: %w", err)
+		}
+
+		log.Infof("Pushed CreateMiner message: %s", signed.Cid())
+		log.Infof("Waiting for confirmation")
+
+		mw, err := wapi.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence)
+		if err != nil {
+			return xerrors.Errorf("waiting for createMiner message: %w", err)
+		}
+
+		if mw.Receipt.ExitCode != 0 {
+			return xerrors.Errorf("create miner failed: exit code %d", mw.Receipt.ExitCode)
+		}
+
+		var retval power6.CreateMinerReturn
+		if err := retval.UnmarshalCBOR(bytes.NewReader(mw.Receipt.Return)); err != nil {
+			return err
+		}
+
+		log.Infof("New miners address is: %s (%s)", retval.IDAddress, retval.RobustAddress)
+
+		return nil
 	},
 }

@@ -148,6 +148,15 @@ func printMessage(cctx *cli.Context, msg *types.Message) error {

 	fmt.Println("Params:", p)

+	if msg, err := messageFromBytes(cctx, msg.Params); err == nil {
+		fmt.Println("---")
+		color.Red("Params message:")
+
+		if err := printMessage(cctx, msg.VMMessage()); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

@@ -2,11 +2,14 @@ package main

 import (
 	"bytes"
+	"context"
 	"encoding/base64"
+	"encoding/binary"
 	"fmt"
 	"image"
 	"image/color"
 	"image/png"
+	"io"
 	"os"
 	"sort"
 	"strconv"
@@ -23,6 +26,7 @@ import (

 	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"

+	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -38,6 +42,7 @@ var sectorsCmd = &cli.Command{
 		terminateSectorCmd,
 		terminateSectorPenaltyEstimationCmd,
 		visAllocatedSectorsCmd,
+		dumpRLESectorCmd,
 	},
 }

@@ -275,6 +280,113 @@ var terminateSectorPenaltyEstimationCmd = &cli.Command{
 	},
 }

+func activeMiners(ctx context.Context, api v0api.FullNode) ([]address.Address, error) {
+	miners, err := api.StateListMiners(ctx, types.EmptyTSK)
+	if err != nil {
+		return nil, err
+	}
+	powCache := make(map[address.Address]types.BigInt)
+	var lk sync.Mutex
+	parmap.Par(32, miners, func(a address.Address) {
+		pow, err := api.StateMinerPower(ctx, a, types.EmptyTSK)
+
+		lk.Lock()
+		if err == nil {
+			powCache[a] = pow.MinerPower.QualityAdjPower
+		} else {
+			powCache[a] = types.NewInt(0)
+		}
+		lk.Unlock()
+	})
+	sort.Slice(miners, func(i, j int) bool {
+		return powCache[miners[i]].GreaterThan(powCache[miners[j]])
+	})
+	n := sort.Search(len(miners), func(i int) bool {
+		pow := powCache[miners[i]]
+		return pow.IsZero()
+	})
+	return append(miners[0:0:0], miners[:n]...), nil
+}
+
+var dumpRLESectorCmd = &cli.Command{
+	Name:  "dump-rles",
+	Usage: "Dump AllocatedSectors RLEs from miners passed as arguments as run lengths in uint64 LE format.\nIf no arguments are passed, dumps all active miners in the state tree.",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+		var miners []address.Address
+		if cctx.NArg() == 0 {
+			miners, err = activeMiners(ctx, api)
+			if err != nil {
+				return xerrors.Errorf("getting active miners: %w", err)
+			}
+		} else {
+			for _, mS := range cctx.Args().Slice() {
+				mA, err := address.NewFromString(mS)
+				if err != nil {
+					return xerrors.Errorf("parsing address '%s': %w", mS, err)
+				}
+				miners = append(miners, mA)
+			}
+		}
+		wbuf := make([]byte, 8)
+		buf := &bytes.Buffer{}
+
+		for i := 0; i < len(miners); i++ {
+			buf.Reset()
+			err := func() error {
+				state, err := api.StateReadState(ctx, miners[i], types.EmptyTSK)
+				if err != nil {
+					return xerrors.Errorf("getting state: %+v", err)
+				}
+				allocSString := state.State.(map[string]interface{})["AllocatedSectors"].(map[string]interface{})["/"].(string)
+
+				allocCid, err := cid.Decode(allocSString)
+				if err != nil {
+					return xerrors.Errorf("decoding cid: %+v", err)
+				}
+				rle, err := api.ChainReadObj(ctx, allocCid)
+				if err != nil {
+					return xerrors.Errorf("reading AllocatedSectors: %+v", err)
+				}
+
+				var bf bitfield.BitField
+				err = bf.UnmarshalCBOR(bytes.NewReader(rle))
+				if err != nil {
+					return xerrors.Errorf("decoding bitfield: %w", err)
+				}
+				ri, err := bf.RunIterator()
+				if err != nil {
+					return xerrors.Errorf("creating iterator: %w", err)
+				}
+
+				for ri.HasNext() {
+					run, err := ri.NextRun()
+					if err != nil {
+						return xerrors.Errorf("getting run: %w", err)
+					}
+					binary.LittleEndian.PutUint64(wbuf, run.Len)
+					buf.Write(wbuf)
+				}
+				_, err = io.Copy(os.Stdout, buf)
+				if err != nil {
+					return xerrors.Errorf("copy: %w", err)
+				}
+
+				return nil
+			}()
+			if err != nil {
+				log.Errorf("miner %d: %s: %+v", i, miners[i], err)
+			}
+		}
+		return nil
+	},
+}
+
 var visAllocatedSectorsCmd = &cli.Command{
 	Name:  "vis-allocated",
 	Usage: "Produces a html with visualisation of allocated sectors",
@@ -287,32 +399,10 @@ var visAllocatedSectorsCmd = &cli.Command{
 		ctx := lcli.ReqContext(cctx)
 		var miners []address.Address
 		if cctx.NArg() == 0 {
-			miners, err = api.StateListMiners(ctx, types.EmptyTSK)
+			miners, err = activeMiners(ctx, api)
 			if err != nil {
-				return err
+				return xerrors.Errorf("getting active miners: %w", err)
 			}
-			powCache := make(map[address.Address]types.BigInt)
-			var lk sync.Mutex
-			parmap.Par(32, miners, func(a address.Address) {
-				pow, err := api.StateMinerPower(ctx, a, types.EmptyTSK)
-
-				lk.Lock()
-				if err == nil {
-					powCache[a] = pow.MinerPower.QualityAdjPower
-				} else {
-					powCache[a] = types.NewInt(0)
-				}
-				lk.Unlock()
-			})
-			sort.Slice(miners, func(i, j int) bool {
-				return powCache[miners[i]].GreaterThan(powCache[miners[j]])
-			})
-			n := sort.Search(len(miners), func(i int) bool {
-				pow := powCache[miners[i]]
-				log.Infof("pow @%d = %s", i, pow)
-				return pow.IsZero()
-			})
-			miners = miners[:n]
 		} else {
 			for _, mS := range cctx.Args().Slice() {
 				mA, err := address.NewFromString(mS)
@@ -2,7 +2,7 @@ version: '3'

 services:
  influxdb:
-    image: influxdb:latest
+    image: influxdb:1.8
    container_name: influxdb
    ports:
      - "18086:8086"
@@ -2,18 +2,36 @@ package main

 import (
 	"context"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
+	"time"

+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/build"
 	lcli "github.com/filecoin-project/lotus/cli"
-	"github.com/filecoin-project/lotus/tools/stats"
+	"github.com/filecoin-project/lotus/tools/stats/influx"
+	"github.com/filecoin-project/lotus/tools/stats/ipldstore"
+	"github.com/filecoin-project/lotus/tools/stats/metrics"
+	"github.com/filecoin-project/lotus/tools/stats/points"
+	"github.com/filecoin-project/lotus/tools/stats/sync"

 	logging "github.com/ipfs/go-log/v2"
 	"github.com/urfave/cli/v2"
+
+	"contrib.go.opencensus.io/exporter/prometheus"
+	stats "go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
 )

 var log = logging.Logger("stats")

+func init() {
+	if err := view.Register(metrics.DefaultViews...); err != nil {
+		log.Fatal(err)
+	}
+}
+
 func main() {
 	local := []*cli.Command{
 		runCmd,
@@ -37,7 +55,7 @@ func main() {
 			},
 		},
 		Before: func(cctx *cli.Context) error {
-			return logging.SetLogLevel("stats", cctx.String("log-level"))
+			return logging.SetLogLevelRegex("stats/*", cctx.String("log-level"))
 		},
 		Commands: local,
 	}
@@ -104,6 +122,12 @@ var runCmd = &cli.Command{
 			Usage:   "do not wait for chain sync to complete",
 			Value:   false,
 		},
+		&cli.IntFlag{
+			Name:    "ipld-store-cache-size",
+			Usage:   "size of lru cache for ChainReadObj",
+			EnvVars: []string{"LOTUS_STATS_IPLD_STORE_CACHE_SIZE"},
+			Value:   2 << 15,
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		ctx := context.Background()
@@ -118,30 +142,35 @@ var runCmd = &cli.Command{
 		influxPasswordFlag := cctx.String("influx-password")
 		influxDatabaseFlag := cctx.String("influx-database")

+		ipldStoreCacheSizeFlag := cctx.Int("ipld-store-cache-size")
+
 		log.Infow("opening influx client", "hostname", influxHostnameFlag, "username", influxUsernameFlag, "database", influxDatabaseFlag)

-		influx, err := stats.InfluxClient(influxHostnameFlag, influxUsernameFlag, influxPasswordFlag)
+		influxClient, err := influx.NewClient(influxHostnameFlag, influxUsernameFlag, influxPasswordFlag)
 		if err != nil {
-			log.Fatal(err)
+			return err
 		}

+		exporter, err := prometheus.NewExporter(prometheus.Options{
+			Namespace: "lotus_stats",
+		})
+		if err != nil {
+			return err
+		}
+
+		go func() {
+			http.Handle("/metrics", exporter)
+			if err := http.ListenAndServe(":6688", nil); err != nil {
+				log.Errorw("failed to start http server", "err", err)
+			}
+		}()
+
 		if resetFlag {
-			if err := stats.ResetDatabase(influx, influxDatabaseFlag); err != nil {
-				log.Fatal(err)
+			if err := influx.ResetDatabase(influxClient, influxDatabaseFlag); err != nil {
+				return err
 			}
 		}

-		height := int64(heightFlag)
-
-		if !resetFlag && height == 0 {
-			h, err := stats.GetLastRecordedHeight(influx, influxDatabaseFlag)
-			if err != nil {
-				log.Info(err)
-			}
-
-			height = h
-		}
-
 		api, closer, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -149,12 +178,89 @@ var runCmd = &cli.Command{
 		defer closer()

 		if !noSyncFlag {
-			if err := stats.WaitForSyncComplete(ctx, api); err != nil {
-				log.Fatal(err)
+			if err := sync.SyncWait(ctx, api); err != nil {
+				return err
 			}
 		}

-		stats.Collect(ctx, api, influx, influxDatabaseFlag, height, headLagFlag)
+		gtp, err := api.ChainGetGenesis(ctx)
+		if err != nil {
+			return err
+		}
+
+		genesisTime := time.Unix(int64(gtp.MinTimestamp()), 0)
+
+		// When height is set to `0` we will resume from the best height we can.
+		// The goal is to ensure we have data in the last 60 tipsets
+		height := int64(heightFlag)
+		if !resetFlag && height == 0 {
+			lastHeight, err := influx.GetLastRecordedHeight(influxClient, influxDatabaseFlag)
+			if err != nil {
+				return err
+			}
+
+			sinceGenesis := build.Clock.Now().Sub(genesisTime)
+			expectedHeight := int64(sinceGenesis.Seconds()) / int64(build.BlockDelaySecs)
+
+			startOfWindowHeight := expectedHeight - 60
+
+			if lastHeight > startOfWindowHeight {
+				height = lastHeight
+			} else {
+				height = startOfWindowHeight
+			}
+
+			ts, err := api.ChainHead(ctx)
+			if err != nil {
+				return err
+			}
+
+			headHeight := int64(ts.Height())
+			if headHeight < height {
+				height = headHeight
+			}
+		}
+
+		go func() {
+			t := time.NewTicker(time.Second)
+
+			for {
+				select {
+				case <-t.C:
+					sinceGenesis := build.Clock.Now().Sub(genesisTime)
+					expectedHeight := int64(sinceGenesis.Seconds()) / int64(build.BlockDelaySecs)
+
+					stats.Record(ctx, metrics.TipsetCollectionHeightExpected.M(expectedHeight))
+				}
+			}
+		}()
+
+		store, err := ipldstore.NewApiIpldStore(ctx, api, ipldStoreCacheSizeFlag)
+		if err != nil {
+			return err
+		}
+
+		collector, err := points.NewChainPointCollector(ctx, store, api)
+		if err != nil {
+			return err
+		}
+
+		tipsets, err := sync.BufferedTipsetChannel(ctx, api, abi.ChainEpoch(height), headLagFlag)
+		if err != nil {
+			return err
+		}
+
+		wq := influx.NewWriteQueue(ctx, influxClient)
+		defer wq.Close()
+
+		for tipset := range tipsets {
+			if nb, err := collector.Collect(ctx, tipset); err != nil {
+				log.Warnw("failed to collect points", "err", err)
+			} else {
+				nb.SetDatabase(influxDatabaseFlag)
+				wq.AddBatch(nb)
+			}
+		}

 		return nil
 	},
@@ -103,6 +103,7 @@ services:
  #     - FULLNODE_API_INFO=/dns/lotus/tcp/1234/http
  #     - LOTUS_JAEGER_AGENT_HOST=jaeger
  #     - LOTUS_JAEGER_AGENT_PORT=6831
+  #     - DOCKER_LOTUS_MINER_INIT=true
  #   deploy:
  #     restart_policy:
  #       condition: on-failure
@@ -61,6 +61,7 @@
  * [MarketPendingDeals](#MarketPendingDeals)
  * [MarketPublishPendingDeals](#MarketPublishPendingDeals)
  * [MarketRestartDataTransfer](#MarketRestartDataTransfer)
+  * [MarketRetryPublishDeal](#MarketRetryPublishDeal)
  * [MarketSetAsk](#MarketSetAsk)
  * [MarketSetRetrievalAsk](#MarketSetRetrievalAsk)
 * [Mining](#Mining)
@@ -949,6 +950,22 @@ Inputs:

 Response: `{}`

+### MarketRetryPublishDeal
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  }
+]
+```
+
+Response: `{}`
+
 ### MarketSetAsk


@@ -2131,7 +2148,9 @@ Inputs:
    "Weight": 42,
    "MaxStorage": 42,
    "CanSeal": true,
-    "CanStore": true
+    "CanStore": true,
+    "Groups": null,
+    "AllowTo": null
  },
  {
    "Capacity": 9,
@@ -2241,7 +2260,9 @@ Response:
  "Weight": 42,
  "MaxStorage": 42,
  "CanSeal": true,
-  "CanStore": true
+  "CanStore": true,
+  "Groups": null,
+  "AllowTo": null
 }
 ```

@@ -2432,18 +2453,595 @@ Response:
      "IgnoreResources": false,
      "Resources": {
        "MemPhysical": 274877906944,
+        "MemUsed": 2147483648,
        "MemSwap": 128849018880,
-        "MemReserved": 2147483648,
+        "MemSwapUsed": 2147483648,
        "CPUs": 64,
        "GPUs": [
          "aGPU 1337"
-        ]
+        ],
+        "Resources": {
+          "seal/v0/addpiece": {
+            "0": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "1": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "2": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "3": {
+              "MinMemory": 4294967296,
+              "MaxMemory": 4294967296,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "4": {
+              "MinMemory": 8589934592,
+              "MaxMemory": 8589934592,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "5": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "6": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "7": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "8": {
+              "MinMemory": 4294967296,
+              "MaxMemory": 4294967296,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "9": {
+              "MinMemory": 8589934592,
+              "MaxMemory": 8589934592,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            }
+          },
+          "seal/v0/commit/1": {
+            "0": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "1": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "2": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "3": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "4": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "5": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "6": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "7": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "8": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "9": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            }
+          },
+          "seal/v0/commit/2": {
+            "0": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 1,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "1": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 1,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "2": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1610612736,
+              "GPUUtilization": 1,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10737418240
+            },
+            "3": {
+              "MinMemory": 32212254720,
+              "MaxMemory": 161061273600,
+              "GPUUtilization": 1,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 6,
+              "BaseMinMemory": 34359738368
+            },
+            "4": {
+              "MinMemory": 64424509440,
+              "MaxMemory": 204010946560,
+              "GPUUtilization": 1,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 6,
+              "BaseMinMemory": 68719476736
+            },
+            "5": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 1,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "6": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 1,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "7": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1610612736,
+              "GPUUtilization": 1,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10737418240
+            },
+            "8": {
+              "MinMemory": 32212254720,
+              "MaxMemory": 161061273600,
+              "GPUUtilization": 1,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 6,
+              "BaseMinMemory": 34359738368
+            },
+            "9": {
+              "MinMemory": 64424509440,
+              "MaxMemory": 204010946560,
+              "GPUUtilization": 1,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 6,
+              "BaseMinMemory": 68719476736
+            }
+          },
+          "seal/v0/fetch": {
+            "0": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            },
+            "1": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            },
+            "2": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            },
+            "3": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            },
+            "4": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            },
+            "5": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            },
+            "6": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            },
+            "7": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            },
+            "8": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            },
+            "9": {
+              "MinMemory": 1048576,
+              "MaxMemory": 1048576,
+              "GPUUtilization": 0,
+              "MaxParallelism": 0,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 0
+            }
+          },
+          "seal/v0/precommit/1": {
+            "0": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "1": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "2": {
+              "MinMemory": 805306368,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1048576
+            },
+            "3": {
+              "MinMemory": 60129542144,
+              "MaxMemory": 68719476736,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10485760
+            },
+            "4": {
+              "MinMemory": 120259084288,
+              "MaxMemory": 137438953472,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10485760
+            },
+            "5": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "6": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "7": {
+              "MinMemory": 805306368,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1048576
+            },
+            "8": {
+              "MinMemory": 60129542144,
+              "MaxMemory": 68719476736,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10485760
+            },
+            "9": {
+              "MinMemory": 120259084288,
+              "MaxMemory": 137438953472,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10485760
+            }
+          },
+          "seal/v0/precommit/2": {
+            "0": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "1": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "2": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1610612736,
+              "GPUUtilization": 0,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "3": {
+              "MinMemory": 16106127360,
+              "MaxMemory": 16106127360,
+              "GPUUtilization": 1,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 6,
+              "BaseMinMemory": 1073741824
+            },
+            "4": {
+              "MinMemory": 32212254720,
+              "MaxMemory": 32212254720,
+              "GPUUtilization": 1,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 6,
+              "BaseMinMemory": 1073741824
+            },
+            "5": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "6": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "7": {
+              "MinMemory": 1073741824,
+              "MaxMemory": 1610612736,
+              "GPUUtilization": 0,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1073741824
+            },
+            "8": {
+              "MinMemory": 16106127360,
+              "MaxMemory": 16106127360,
+              "GPUUtilization": 1,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 6,
+              "BaseMinMemory": 1073741824
+            },
+            "9": {
+              "MinMemory": 32212254720,
+              "MaxMemory": 32212254720,
+              "GPUUtilization": 1,
+              "MaxParallelism": -1,
+              "MaxParallelismGPU": 6,
+              "BaseMinMemory": 1073741824
+            }
+          },
+          "seal/v0/unseal": {
+            "0": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "1": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "2": {
+              "MinMemory": 805306368,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1048576
+            },
+            "3": {
+              "MinMemory": 60129542144,
+              "MaxMemory": 68719476736,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10485760
+            },
+            "4": {
+              "MinMemory": 120259084288,
+              "MaxMemory": 137438953472,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10485760
+            },
+            "5": {
+              "MinMemory": 2048,
+              "MaxMemory": 2048,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 2048
+            },
+            "6": {
+              "MinMemory": 8388608,
+              "MaxMemory": 8388608,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 8388608
+            },
+            "7": {
+              "MinMemory": 805306368,
+              "MaxMemory": 1073741824,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 1048576
+            },
+            "8": {
+              "MinMemory": 60129542144,
+              "MaxMemory": 68719476736,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10485760
+            },
+            "9": {
+              "MinMemory": 120259084288,
+              "MaxMemory": 137438953472,
+              "GPUUtilization": 0,
+              "MaxParallelism": 1,
+              "MaxParallelismGPU": 0,
+              "BaseMinMemory": 10485760
+            }
+          }
+        }
      }
    },
    "Enabled": true,
    "MemUsedMin": 0,
    "MemUsedMax": 0,
-    "GpuUsed": false,
+    "GpuUsed": 0,
    "CpuUse": 0
  }
 }
@@ -92,10 +92,587 @@ Response:
  "IgnoreResources": true,
  "Resources": {
    "MemPhysical": 42,
+    "MemUsed": 42,
    "MemSwap": 42,
-    "MemReserved": 42,
+    "MemSwapUsed": 42,
    "CPUs": 42,
-    "GPUs": null
+    "GPUs": null,
+    "Resources": {
+      "seal/v0/addpiece": {
+        "0": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "1": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "2": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "3": {
+          "MinMemory": 4294967296,
+          "MaxMemory": 4294967296,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "4": {
+          "MinMemory": 8589934592,
+          "MaxMemory": 8589934592,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "5": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "6": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "7": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "8": {
+          "MinMemory": 4294967296,
+          "MaxMemory": 4294967296,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "9": {
+          "MinMemory": 8589934592,
+          "MaxMemory": 8589934592,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        }
+      },
+      "seal/v0/commit/1": {
+        "0": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "1": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "2": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "3": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "4": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "5": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "6": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "7": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "8": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "9": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        }
+      },
+      "seal/v0/commit/2": {
+        "0": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 1,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "1": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 1,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "2": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1610612736,
+          "GPUUtilization": 1,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10737418240
+        },
+        "3": {
+          "MinMemory": 32212254720,
+          "MaxMemory": 161061273600,
+          "GPUUtilization": 1,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 6,
+          "BaseMinMemory": 34359738368
+        },
+        "4": {
+          "MinMemory": 64424509440,
+          "MaxMemory": 204010946560,
+          "GPUUtilization": 1,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 6,
+          "BaseMinMemory": 68719476736
+        },
+        "5": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 1,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "6": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 1,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "7": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1610612736,
+          "GPUUtilization": 1,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10737418240
+        },
+        "8": {
+          "MinMemory": 32212254720,
+          "MaxMemory": 161061273600,
+          "GPUUtilization": 1,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 6,
+          "BaseMinMemory": 34359738368
+        },
+        "9": {
+          "MinMemory": 64424509440,
+          "MaxMemory": 204010946560,
+          "GPUUtilization": 1,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 6,
+          "BaseMinMemory": 68719476736
+        }
+      },
+      "seal/v0/fetch": {
+        "0": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        },
+        "1": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        },
+        "2": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        },
+        "3": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        },
+        "4": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        },
+        "5": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        },
+        "6": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        },
+        "7": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        },
+        "8": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        },
+        "9": {
+          "MinMemory": 1048576,
+          "MaxMemory": 1048576,
+          "GPUUtilization": 0,
+          "MaxParallelism": 0,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 0
+        }
+      },
+      "seal/v0/precommit/1": {
+        "0": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "1": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "2": {
+          "MinMemory": 805306368,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1048576
+        },
+        "3": {
+          "MinMemory": 60129542144,
+          "MaxMemory": 68719476736,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10485760
+        },
+        "4": {
+          "MinMemory": 120259084288,
+          "MaxMemory": 137438953472,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10485760
+        },
+        "5": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "6": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "7": {
+          "MinMemory": 805306368,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1048576
+        },
+        "8": {
+          "MinMemory": 60129542144,
+          "MaxMemory": 68719476736,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10485760
+        },
+        "9": {
+          "MinMemory": 120259084288,
+          "MaxMemory": 137438953472,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10485760
+        }
+      },
+      "seal/v0/precommit/2": {
+        "0": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "1": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "2": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1610612736,
+          "GPUUtilization": 0,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "3": {
+          "MinMemory": 16106127360,
+          "MaxMemory": 16106127360,
+          "GPUUtilization": 1,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 6,
+          "BaseMinMemory": 1073741824
+        },
+        "4": {
+          "MinMemory": 32212254720,
+          "MaxMemory": 32212254720,
+          "GPUUtilization": 1,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 6,
+          "BaseMinMemory": 1073741824
+        },
+        "5": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "6": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "7": {
+          "MinMemory": 1073741824,
+          "MaxMemory": 1610612736,
+          "GPUUtilization": 0,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1073741824
+        },
+        "8": {
+          "MinMemory": 16106127360,
+          "MaxMemory": 16106127360,
+          "GPUUtilization": 1,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 6,
+          "BaseMinMemory": 1073741824
+        },
+        "9": {
+          "MinMemory": 32212254720,
+          "MaxMemory": 32212254720,
+          "GPUUtilization": 1,
+          "MaxParallelism": -1,
+          "MaxParallelismGPU": 6,
+          "BaseMinMemory": 1073741824
+        }
+      },
+      "seal/v0/unseal": {
+        "0": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "1": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "2": {
+          "MinMemory": 805306368,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1048576
+        },
+        "3": {
+          "MinMemory": 60129542144,
+          "MaxMemory": 68719476736,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10485760
+        },
+        "4": {
+          "MinMemory": 120259084288,
+          "MaxMemory": 137438953472,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10485760
+        },
+        "5": {
+          "MinMemory": 2048,
+          "MaxMemory": 2048,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 2048
+        },
+        "6": {
+          "MinMemory": 8388608,
+          "MaxMemory": 8388608,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 8388608
+        },
+        "7": {
+          "MinMemory": 805306368,
+          "MaxMemory": 1073741824,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 1048576
+        },
+        "8": {
+          "MinMemory": 60129542144,
+          "MaxMemory": 68719476736,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10485760
+        },
+        "9": {
+          "MinMemory": 120259084288,
+          "MaxMemory": 137438953472,
+          "GPUUtilization": 0,
+          "MaxParallelism": 1,
+          "MaxParallelismGPU": 0,
+          "BaseMinMemory": 10485760
+        }
+      }
+    }
  }
 }
 ```
@@ -1269,7 +1269,8 @@ Response:
    "Stages": {
      "Stages": null
    }
-  }
+  },
+  "Event": 5
 }
 ```

@@ -41,6 +41,7 @@
  * [ClientDataTransferUpdates](#ClientDataTransferUpdates)
  * [ClientDealPieceCID](#ClientDealPieceCID)
  * [ClientDealSize](#ClientDealSize)
+  * [ClientExport](#ClientExport)
  * [ClientFindData](#ClientFindData)
  * [ClientGenCar](#ClientGenCar)
  * [ClientGetDealInfo](#ClientGetDealInfo)
@@ -59,7 +60,7 @@
  * [ClientRestartDataTransfer](#ClientRestartDataTransfer)
  * [ClientRetrieve](#ClientRetrieve)
  * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds)
-  * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents)
+  * [ClientRetrieveWait](#ClientRetrieveWait)
  * [ClientStartDeal](#ClientStartDeal)
  * [ClientStatelessDeal](#ClientStatelessDeal)
 * [Create](#Create)
@@ -108,6 +109,7 @@
  * [MsigApprove](#MsigApprove)
  * [MsigApproveTxnHash](#MsigApproveTxnHash)
  * [MsigCancel](#MsigCancel)
+  * [MsigCancelTxnHash](#MsigCancelTxnHash)
  * [MsigCreate](#MsigCreate)
  * [MsigGetAvailableBalance](#MsigGetAvailableBalance)
  * [MsigGetPending](#MsigGetPending)
@@ -1054,6 +1056,32 @@ Response:
 }
 ```

+### ClientExport
+ClientExport exports a file stored in the local filestore to a system file
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  {
+    "Root": {
+      "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+    },
+    "DAGs": null,
+    "FromLocalCAR": "string value",
+    "DealID": 5
+  },
+  {
+    "Path": "string value",
+    "IsCAR": true
+  }
+]
+```
+
+Response: `{}`
+
 ### ClientFindData
 ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer).

@@ -1281,7 +1309,8 @@ Response:
    "Stages": {
      "Stages": null
    }
-  }
+  },
+  "Event": 5
 }
 ```

@@ -1481,9 +1510,8 @@ Inputs:
      "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
    },
    "Piece": null,
-    "DatamodelPathSelector": "Links/21/Hash/Links/42/Hash",
+    "DataSelector": "Links/21/Hash/Links/42/Hash",
    "Size": 42,
-    "FromLocalCAR": "string value",
    "Total": "0",
    "UnsealPrice": "0",
    "PaymentInterval": 42,
@@ -1495,15 +1523,16 @@ Inputs:
      "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf",
      "PieceCID": null
    }
-  },
-  {
-    "Path": "string value",
-    "IsCAR": true
  }
 ]
 ```

-Response: `{}`
+Response:
+```json
+{
+  "DealID": 5
+}
+```

 ### ClientRetrieveTryRestartInsufficientFunds
 ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel
@@ -1521,9 +1550,8 @@ Inputs:

 Response: `{}`

-### ClientRetrieveWithEvents
-ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
-of status updates.
+### ClientRetrieveWait
+ClientRetrieveWait waits for retrieval to be complete


 Perms: admin
@@ -1531,43 +1559,11 @@ Perms: admin
 Inputs:
 ```json
 [
-  {
-    "Root": {
-      "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
-    },
-    "Piece": null,
-    "DatamodelPathSelector": "Links/21/Hash/Links/42/Hash",
-    "Size": 42,
-    "FromLocalCAR": "string value",
-    "Total": "0",
-    "UnsealPrice": "0",
-    "PaymentInterval": 42,
-    "PaymentIntervalIncrease": 42,
-    "Client": "f01234",
-    "Miner": "f01234",
-    "MinerPeer": {
-      "Address": "f01234",
-      "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf",
-      "PieceCID": null
-    }
-  },
-  {
-    "Path": "string value",
-    "IsCAR": true
-  }
+  5
 ]
 ```

-Response:
-```json
-{
-  "Event": 5,
-  "Status": 0,
-  "BytesReceived": 42,
-  "FundsSpent": "0",
-  "Err": "string value"
-}
-```
+Response: `{}`

 ### ClientStartDeal
 ClientStartDeal proposes a deal with a miner.
@@ -2702,6 +2698,44 @@ Response:

 ### MsigCancel
 MsigCancel cancels a previously-proposed multisig message
+It takes the following params: <multisig address>, <proposed transaction ID> <signer address>
+
+
+Perms: sign
+
+Inputs:
+```json
+[
+  "f01234",
+  42,
+  "f01234"
+]
+```
+
+Response:
+```json
+{
+  "Message": {
+    "Version": 42,
+    "To": "f01234",
+    "From": "f01234",
+    "Nonce": 42,
+    "Value": "0",
+    "GasLimit": 9,
+    "GasFeeCap": "0",
+    "GasPremium": "0",
+    "Method": 1,
+    "Params": "Ynl0ZSBhcnJheQ==",
+    "CID": {
+      "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s"
+    }
+  },
+  "ValidNonce": true
+}
+```
+
+### MsigCancelTxnHash
+MsigCancel cancels a previously-proposed multisig message
 It takes the following params: <multisig address>, <proposed transaction ID>, <recipient address>, <value to transfer>,
 <sender address of the cancel msg>, <method to call in the proposed message>, <params to include in the proposed message>

@@ -7,7 +7,7 @@ USAGE:
   lotus-miner [global options] command [command options] [arguments...]

 VERSION:
-   1.13.0
+   1.13.2

 COMMANDS:
   init     Initialize a lotus miner repo
@@ -590,7 +590,8 @@ CATEGORY:
   DEVELOPER

 OPTIONS:
-   --help, -h  show help (default: false)
+   --timeout value  duration to wait till fail (default: 30s)
+   --help, -h       show help (default: false)
   
 ```

@@ -629,6 +630,7 @@ COMMANDS:
   reset-blocklist    Remove all entries from the miner's piece CID blocklist
   set-seal-duration  Set the expected time, in minutes, that you expect sealing sectors to take. Deals that start before this duration will be rejected.
   pending-publish    list deals waiting in publish queue
+   retry-publish      retry publishing a deal
   help, h            Shows a list of commands or help for one command

 OPTIONS:
@@ -825,6 +827,19 @@ OPTIONS:
   
 ```

+### lotus-miner storage-deals retry-publish
+```
+NAME:
+   lotus-miner storage-deals retry-publish - retry publishing a deal
+
+USAGE:
+   lotus-miner storage-deals retry-publish [command options] <proposal CID>
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus-miner retrieval-deals
 ```
 NAME:
@@ -1445,7 +1460,8 @@ USAGE:
   lotus-miner pieces list-cids [command options] [arguments...]

 OPTIONS:
-   --help, -h  show help (default: false)
+   --verbose, -v  (default: false)
+   --help, -h     show help (default: false)
   
 ```

@@ -1899,9 +1915,10 @@ USAGE:
   lotus-miner proving check [command options] <deadlineIdx>

 OPTIONS:
-   --only-bad  print only bad sectors (default: false)
-   --slow      run slower checks (default: false)
-   --help, -h  show help (default: false)
+   --only-bad          print only bad sectors (default: false)
+   --slow              run slower checks (default: false)
+   --storage-id value  filter sectors by storage path (path id)
+   --help, -h          show help (default: false)
   
 ```

@@ -1967,6 +1984,8 @@ OPTIONS:
   --seal               (for init) use path for sealing (default: false)
   --store              (for init) use path for long-term storage (default: false)
   --max-storage value  (for init) limit storage space for sectors (expensive for very large paths!)
+   --groups value       path group names
+   --allow-to value     path groups allowed to pull data from this path (allow all if not specified)
   --help, -h           show help (default: false)
   
 ```
@@ -7,7 +7,7 @@ USAGE:
   lotus-worker [global options] command [command options] [arguments...]

 VERSION:
-   1.13.0
+   1.13.2

 COMMANDS:
   run         Start lotus worker
@@ -15,6 +15,7 @@ COMMANDS:
   storage     manage sector storage
   set         Manage worker settings
   wait-quiet  Block until all running tasks exit
+   resources   Manage resource table overrides
   tasks       Manage task processing
   help, h     Shows a list of commands or help for one command

@@ -94,6 +95,8 @@ OPTIONS:
   --seal               (for init) use path for sealing (default: false)
   --store              (for init) use path for long-term storage (default: false)
   --max-storage value  (for init) limit storage space for sectors (expensive for very large paths!)
+   --groups value       path group names
+   --allow-to value     path groups allowed to pull data from this path (allow all if not specified)
   --help, -h           show help (default: false)
   
 ```
@@ -125,6 +128,21 @@ OPTIONS:
   
 ```

+## lotus-worker resources
+```
+NAME:
+   lotus-worker resources - Manage resource table overrides
+
+USAGE:
+   lotus-worker resources [command options] [arguments...]
+
+OPTIONS:
+   --all       print all resource envvars (default: false)
+   --default   print default resource envvars (default: false)
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus-worker tasks
 ```
 NAME:
@@ -7,7 +7,7 @@ USAGE:
   lotus [global options] command [command options] [arguments...]

 VERSION:
-   1.13.0
+   1.13.2

 COMMANDS:
   daemon   Start a lotus daemon process
@@ -426,6 +426,8 @@ COMMANDS:
   RETRIEVAL:
     find              Find data in the network
     retrieve          Retrieve data from network
+     cat               Show data from network
+     ls                List object links
     cancel-retrieval  Cancel a retrieval deal by deal ID; this also cancels the associated transfer
     list-retrievals   List retrieval market deals
   STORAGE:
@@ -544,12 +546,94 @@ USAGE:
 CATEGORY:
   RETRIEVAL

+DESCRIPTION:
+   Retrieve data from the Filecoin network.
+
+The retrieve command will attempt to find a provider make a retrieval deal with
+them. In case a provider can't be found, it can be specified with the --provider
+flag.
+
+By default the data will be interpreted as DAG-PB UnixFSv1 File. Alternatively
+a CAR file containing the raw IPLD graph can be exported by setting the --car
+flag.
+
+Partial Retrieval:
+
+The --data-selector flag can be used to specify a sub-graph to fetch. The
+selector can be specified as either IPLD datamodel text-path selector, or IPLD
+json selector.
+
+In case of unixfs retrieval, the selector must point at a single root node, and
+match the entire graph under that node.
+
+In case of CAR retrieval, the selector must have one common "sub-root" node.
+
+Examples:
+
+- Retrieve a file by CID
+  $ lotus client retrieve Qm... my-file.txt
+
+- Retrieve a file by CID from f0123
+  $ lotus client retrieve --provider f0123 Qm... my-file.txt
+
+- Retrieve a first file from a specified directory
+  $ lotus client retrieve --data-selector /Links/0/Hash Qm... my-file.txt
+
+
 OPTIONS:
+   --car                                                   Export to a car file instead of a regular file (default: false)
+   --data-selector value, --datamodel-path-selector value  IPLD datamodel text-path selector, or IPLD json selector
+   --car-export-merkle-proof                               (requires --data-selector and --car) Export data-selector merkle proof (default: false)
+   --from value                                            address to send transactions from
+   --provider value, --miner value                         provider to use for retrieval, if not present it'll use local discovery
+   --maxPrice value                                        maximum price the client is willing to consider (default: 0 FIL)
+   --pieceCid value                                        require data to be retrieved from a specific Piece CID
+   --allow-local                                           (default: false)
+   --help, -h                                              show help (default: false)
+   
+```
+
+### lotus client cat
+```
+NAME:
+   lotus client cat - Show data from network
+
+USAGE:
+   lotus client cat [command options] [dataCid]
+
+CATEGORY:
+   RETRIEVAL
+
+OPTIONS:
+   --ipld                           list IPLD datamodel links (default: false)
+   --data-selector value            IPLD datamodel text-path selector, or IPLD json selector
   --from value                     address to send transactions from
-   --car                            export to a car file instead of a regular file (default: false)
-   --miner value                    miner address for retrieval, if not present it'll use local discovery
-   --datamodel-path-selector value  a rudimentary (DM-level-only) text-path selector, allowing for sub-selection within a deal
-   --maxPrice value                 maximum price the client is willing to consider (default: 0.01 FIL)
+   --provider value, --miner value  provider to use for retrieval, if not present it'll use local discovery
+   --maxPrice value                 maximum price the client is willing to consider (default: 0 FIL)
+   --pieceCid value                 require data to be retrieved from a specific Piece CID
+   --allow-local                    (default: false)
+   --help, -h                       show help (default: false)
+   
+```
+
+### lotus client ls
+```
+NAME:
+   lotus client ls - List object links
+
+USAGE:
+   lotus client ls [command options] [dataCid]
+
+CATEGORY:
+   RETRIEVAL
+
+OPTIONS:
+   --ipld                           list IPLD datamodel links (default: false)
+   --depth value                    list links recursively up to the specified depth (default: 1)
+   --data-selector value            IPLD datamodel text-path selector, or IPLD json selector
+   --from value                     address to send transactions from
+   --provider value, --miner value  provider to use for retrieval, if not present it'll use local discovery
+   --maxPrice value                 maximum price the client is willing to consider (default: 0 FIL)
   --pieceCid value                 require data to be retrieved from a specific Piece CID
   --allow-local                    (default: false)
   --help, -h                       show help (default: false)
@@ -857,6 +941,7 @@ COMMANDS:
   propose            Propose a multisig transaction
   propose-remove     Propose to remove a signer
   approve            Approve a multisig message
+   cancel             Cancel a multisig message
   add-propose        Propose to add a signer
   add-approve        Approve a message to add a signer
   add-cancel         Cancel a message to add a signer
@@ -952,6 +1037,20 @@ OPTIONS:
   
 ```

+### lotus msig cancel
+```
+NAME:
+   lotus msig cancel - Cancel a multisig message
+
+USAGE:
+   lotus msig cancel [command options] <multisigAddress messageId> [destination value [methodId methodParams]]
+
+OPTIONS:
+   --from value  account to send the cancel message from
+   --help, -h    show help (default: false)
+   
+```
+
 ### lotus msig add-propose
 ```
 NAME:
@@ -1117,12 +1216,12 @@ USAGE:
   lotus filplus command [command options] [arguments...]

 COMMANDS:
-   grant-datacap           give allowance to the specified verified client address
-   list-notaries           list all notaries
-   list-clients            list all verified clients
-   check-client-datacap    check verified client remaining bytes
-   check-notaries-datacap  check notaries remaining bytes
-   help, h                 Shows a list of commands or help for one command
+   grant-datacap         give allowance to the specified verified client address
+   list-notaries         list all notaries
+   list-clients          list all verified clients
+   check-client-datacap  check verified client remaining bytes
+   check-notary-datacap  check a notary's remaining bytes
+   help, h               Shows a list of commands or help for one command

 OPTIONS:
   --help, -h     show help (default: false)
@@ -1183,13 +1282,13 @@ OPTIONS:
   
 ```

-### lotus filplus check-notaries-datacap
+### lotus filplus check-notary-datacap
 ```
 NAME:
-   lotus filplus check-notaries-datacap - check notaries remaining bytes
+   lotus filplus check-notary-datacap - check a notary's remaining bytes

 USAGE:
-   lotus filplus check-notaries-datacap [command options] [arguments...]
+   lotus filplus check-notary-datacap [command options] [arguments...]

 OPTIONS:
   --help, -h  show help (default: false)
@@ -1580,8 +1679,18 @@ OPTIONS:
   --help, -h  show help (default: false)
   
 ```
-# nage
+
+### lotus mpool manage
 ```
+NAME:
+   lotus mpool manage - 
+
+USAGE:
+   lotus mpool manage [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
 ```

 ## lotus state
@@ -2450,7 +2559,8 @@ CATEGORY:
   DEVELOPER

 OPTIONS:
-   --help, -h  show help (default: false)
+   --timeout value  duration to wait till fail (default: 30s)
+   --help, -h       show help (default: false)
   
 ```

@@ -3,17 +3,11 @@
 - [ ] Import new actors
 - [ ] Define upgrade heights in `build/params_`
 - [ ] Generate adapters
-  - [ ] Add the new version in `chain/actors/agen/main.go`
+  - [ ] Update `gen/inlinegen-data.json`
+  - [ ] Update `chain/actors/version.go`
  - [ ] Update adapter code in `chain/actors/builtin` if needed
- [ ] Update `chain/actors/policy/policy.go`
- [ ] Update `chain/actors/version.go`
- [ ] Register in `chain/vm/invoker.go`
- [ ] Register in `chain/vm/mkactor.go`
- [ ] Update `chain/types/state.go`
- [ ] Update `chain/state/statetree.go` (New / Load)
- [ ] Update `chain/stmgr/forks.go`
+  - [ ] Run `make actors-gen`
+- [ ] Update `chain/consensus/filcns/upgrades.go`
  - [ ] Schedule
  - [ ] Migration
- [ ] Update upgrade schedule in `api/test/test.go` and `chain/sync_test.go`
- [ ] Update `NewestNetworkVersion` in `build/params_shared_vals.go`
- [ ] Register in init in `chain/stmgr/utils.go`
+- [ ] Update upgrade schedule in `chain/sync_test.go`
@@ -0,0 +1,12 @@
+//go:build !linux
+// +build !linux
+
+package sectorstorage
+
+func cgroupV1Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	return 0, 0, 0, 0, nil
+}
+
+func cgroupV2Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	return 0, 0, 0, 0, nil
+}
@@ -0,0 +1,117 @@
+//go:build linux
+// +build linux
+
+package sectorstorage
+
+import (
+	"bufio"
+	"bytes"
+	"math"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/cgroups"
+	cgroupv2 "github.com/containerd/cgroups/v2"
+)
+
+func cgroupV2MountPoint() (string, error) {
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close() //nolint
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		fields := bytes.Fields(scanner.Bytes())
+		if len(fields) >= 9 && bytes.Equal(fields[8], []byte("cgroup2")) {
+			return string(fields[4]), nil
+		}
+	}
+	return "", cgroups.ErrMountPointNotExist
+}
+
+func cgroupV1Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	path := cgroups.NestedPath("")
+	if pid := os.Getpid(); pid == 1 {
+		path = cgroups.RootPath
+	}
+	c, err := cgroups.Load(cgroups.SingleSubsystem(cgroups.V1, cgroups.Memory), path)
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	stats, err := c.Stat()
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	if stats.Memory == nil {
+		return 0, 0, 0, 0, nil
+	}
+	if stats.Memory.Usage != nil {
+		memoryMax = stats.Memory.Usage.Limit
+		// Exclude cached files
+		memoryUsed = stats.Memory.Usage.Usage - stats.Memory.InactiveFile - stats.Memory.ActiveFile
+	}
+	if stats.Memory.Swap != nil {
+		swapMax = stats.Memory.Swap.Limit
+		swapUsed = stats.Memory.Swap.Usage
+	}
+	return memoryMax, memoryUsed, swapMax, swapUsed, nil
+}
+
+func cgroupV2MemFromPath(mp, path string) (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	c, err := cgroupv2.LoadManager(mp, path)
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	stats, err := c.Stat()
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	if stats.Memory != nil {
+		memoryMax = stats.Memory.UsageLimit
+		// Exclude memory used caching files
+		memoryUsed = stats.Memory.Usage - stats.Memory.File
+		swapMax = stats.Memory.SwapLimit
+		swapUsed = stats.Memory.SwapUsage
+	}
+
+	return memoryMax, memoryUsed, swapMax, swapUsed, nil
+}
+
+func cgroupV2Mem() (memoryMax, memoryUsed, swapMax, swapUsed uint64, err error) {
+	memoryMax = math.MaxUint64
+	swapMax = math.MaxUint64
+
+	path, err := cgroupv2.PidGroupPath(os.Getpid())
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	mp, err := cgroupV2MountPoint()
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	for path != "/" {
+		cgMemoryMax, cgMemoryUsed, cgSwapMax, cgSwapUsed, err := cgroupV2MemFromPath(mp, path)
+		if err != nil {
+			return 0, 0, 0, 0, err
+		}
+		if cgMemoryMax != 0 && cgMemoryMax < memoryMax {
+			log.Debugf("memory limited by cgroup %s: %v", path, cgMemoryMax)
+			memoryMax = cgMemoryMax
+			memoryUsed = cgMemoryUsed
+		}
+		if cgSwapMax != 0 && cgSwapMax < swapMax {
+			log.Debugf("swap limited by cgroup %s: %v", path, cgSwapMax)
+			swapMax = cgSwapMax
+			swapUsed = cgSwapUsed
+		}
+		path = filepath.Dir(path)
+	}
+
+	return memoryMax, memoryUsed, swapMax, swapUsed, nil
+}
@@ -7,6 +7,9 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
 	"io"
 	"math/bits"
 	"os"
@@ -530,9 +533,19 @@ func (sb *Sealer) SealPreCommit1(ctx context.Context, sector storage.SectorRef,
 	if err != nil {
 		return nil, xerrors.Errorf("presealing sector %d (%s): %w", sector.ID.Number, paths.Unsealed, err)
 	}
-	return p1o, nil
+
+	p1odec := map[string]interface{}{}
+	if err := json.Unmarshal(p1o, &p1odec); err != nil {
+		return nil, xerrors.Errorf("unmarshaling pc1 output: %w", err)
+	}
+
+	p1odec["_lotus_SealRandomness"] = ticket
+
+	return json.Marshal(&p1odec)
 }

+var PC2CheckRounds = 3
+
 func (sb *Sealer) SealPreCommit2(ctx context.Context, sector storage.SectorRef, phase1Out storage.PreCommit1Out) (storage.SectorCids, error) {
 	paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, 0, storiface.PathSealing)
 	if err != nil {
@@ -545,6 +558,50 @@ func (sb *Sealer) SealPreCommit2(ctx context.Context, sector storage.SectorRef,
 		return storage.SectorCids{}, xerrors.Errorf("presealing sector %d (%s): %w", sector.ID.Number, paths.Unsealed, err)
 	}

+	ssize, err := sector.ProofType.SectorSize()
+	if err != nil {
+		return storage.SectorCids{}, xerrors.Errorf("get ssize: %w", err)
+	}
+
+	p1odec := map[string]interface{}{}
+	if err := json.Unmarshal(phase1Out, &p1odec); err != nil {
+		return storage.SectorCids{}, xerrors.Errorf("unmarshaling pc1 output: %w", err)
+	}
+
+	var ticket abi.SealRandomness
+	ti, found := p1odec["_lotus_SealRandomness"]
+
+	if found {
+		ticket, err = base64.StdEncoding.DecodeString(ti.(string))
+		if err != nil {
+			return storage.SectorCids{}, xerrors.Errorf("decoding ticket: %w", err)
+		}
+
+		for i := 0; i < PC2CheckRounds; i++ {
+			var sd [32]byte
+			_, _ = rand.Read(sd[:])
+
+			_, err := ffi.SealCommitPhase1(
+				sector.ProofType,
+				sealedCID,
+				unsealedCID,
+				paths.Cache,
+				paths.Sealed,
+				sector.ID.Number,
+				sector.ID.Miner,
+				ticket,
+				sd[:],
+				[]abi.PieceInfo{{Size: abi.PaddedPieceSize(ssize), PieceCID: unsealedCID}},
+			)
+			if err != nil {
+				log.Warn("checking PreCommit failed: ", err)
+				log.Warnf("num:%d tkt:%v seed:%v sealedCID:%v, unsealedCID:%v", sector.ID.Number, ticket, sd[:], sealedCID, unsealedCID)
+
+				return storage.SectorCids{}, xerrors.Errorf("checking PreCommit failed: %w", err)
+			}
+		}
+	}
+
 	return storage.SectorCids{
 		Unsealed: unsealedCID,
 		Sealed:   sealedCID,
@@ -8,7 +8,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 )

-var MTTresh = uint64(32 << 20)
+var MTTresh = uint64(512 << 10)

 func mtChunkCount(usz abi.PaddedPieceSize) uint64 {
 	threads := (uint64(usz)) / MTTresh
@@ -16,13 +16,21 @@ type unpadReader struct {
 	work []byte
 }

+func BufSize(sz abi.PaddedPieceSize) int {
+	return int(MTTresh * mtChunkCount(sz))
+}
+
 func NewUnpadReader(src io.Reader, sz abi.PaddedPieceSize) (io.Reader, error) {
+	buf := make([]byte, BufSize(sz))
+
+	return NewUnpadReaderBuf(src, sz, buf)
+}
+
+func NewUnpadReaderBuf(src io.Reader, sz abi.PaddedPieceSize, buf []byte) (io.Reader, error) {
 	if err := sz.Validate(); err != nil {
 		return nil, xerrors.Errorf("bad piece size: %w", err)
 	}

-	buf := make([]byte, MTTresh*mtChunkCount(sz))
-
 	return &unpadReader{
 		src: src,

@@ -1,3 +1,6 @@
+//go:build !windows
+// +build !windows
+
 package fsutil

 import (
@@ -51,13 +51,8 @@ type SectorManager interface {
 	FaultTracker
 }

-type WorkerID uuid.UUID // worker session UUID
 var ClosedWorkerID = uuid.UUID{}

-func (w WorkerID) String() string {
-	return uuid.UUID(w).String()
-}
-
 type Manager struct {
 	ls         stores.LocalStorage
 	storage    *stores.Remote
@@ -588,13 +583,13 @@ func (m *Manager) Remove(ctx context.Context, sector storage.SectorRef) error {

 	var err error

-	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTSealed, true); rerr != nil {
+	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTSealed, true, nil); rerr != nil {
 		err = multierror.Append(err, xerrors.Errorf("removing sector (sealed): %w", rerr))
 	}
-	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTCache, true); rerr != nil {
+	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTCache, true, nil); rerr != nil {
 		err = multierror.Append(err, xerrors.Errorf("removing sector (cache): %w", rerr))
 	}
-	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTUnsealed, true); rerr != nil {
+	if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTUnsealed, true, nil); rerr != nil {
 		err = multierror.Append(err, xerrors.Errorf("removing sector (unsealed): %w", rerr))
 	}

@@ -322,7 +322,7 @@ func TestRestartWorker(t *testing.T) {
 	defer cleanup()

 	localTasks := []sealtasks.TaskType{
-		sealtasks.TTAddPiece, sealtasks.TTPreCommit1, sealtasks.TTCommit1, sealtasks.TTFinalize, sealtasks.TTFetch,
+		sealtasks.TTAddPiece, sealtasks.TTFetch,
 	}

 	wds := datastore.NewMapDatastore()
@@ -332,7 +332,7 @@ func TestRestartWorker(t *testing.T) {
 		return &testExec{apch: arch}, nil
 	}, WorkerConfig{
 		TaskTypes: localTasks,
-	}, stor, lstor, idx, m, statestore.New(wds))
+	}, os.LookupEnv, stor, lstor, idx, m, statestore.New(wds))

 	err := m.AddWorker(ctx, w)
 	require.NoError(t, err)
@@ -368,7 +368,7 @@ func TestRestartWorker(t *testing.T) {
 		return &testExec{apch: arch}, nil
 	}, WorkerConfig{
 		TaskTypes: localTasks,
-	}, stor, lstor, idx, m, statestore.New(wds))
+	}, os.LookupEnv, stor, lstor, idx, m, statestore.New(wds))

 	err = m.AddWorker(ctx, w)
 	require.NoError(t, err)
@@ -404,7 +404,7 @@ func TestReenableWorker(t *testing.T) {
 		return &testExec{apch: arch}, nil
 	}, WorkerConfig{
 		TaskTypes: localTasks,
-	}, stor, lstor, idx, m, statestore.New(wds))
+	}, os.LookupEnv, stor, lstor, idx, m, statestore.New(wds))

 	err := m.AddWorker(ctx, w)
 	require.NoError(t, err)
@@ -453,3 +453,123 @@ func TestReenableWorker(t *testing.T) {
 	i, _ = m.sched.Info(ctx)
 	require.Len(t, i.(SchedDiagInfo).OpenWindows, 2)
 }
+
+func TestResUse(t *testing.T) {
+	logging.SetAllLoggers(logging.LevelDebug)
+
+	ctx, done := context.WithCancel(context.Background())
+	defer done()
+
+	ds := datastore.NewMapDatastore()
+
+	m, lstor, stor, idx, cleanup := newTestMgr(ctx, t, ds)
+	defer cleanup()
+
+	localTasks := []sealtasks.TaskType{
+		sealtasks.TTAddPiece, sealtasks.TTFetch,
+	}
+
+	wds := datastore.NewMapDatastore()
+
+	arch := make(chan chan apres)
+	w := newLocalWorker(func() (ffiwrapper.Storage, error) {
+		return &testExec{apch: arch}, nil
+	}, WorkerConfig{
+		TaskTypes: localTasks,
+	}, func(s string) (string, bool) {
+		return "", false
+	}, stor, lstor, idx, m, statestore.New(wds))
+
+	err := m.AddWorker(ctx, w)
+	require.NoError(t, err)
+
+	sid := storage.SectorRef{
+		ID:        abi.SectorID{Miner: 1000, Number: 1},
+		ProofType: abi.RegisteredSealProof_StackedDrg2KiBV1,
+	}
+
+	go func() {
+		_, err := m.AddPiece(ctx, sid, nil, 1016, strings.NewReader(strings.Repeat("testthis", 127)))
+		require.Error(t, err)
+	}()
+
+l:
+	for {
+		st := m.WorkerStats()
+		require.Len(t, st, 1)
+		for _, w := range st {
+			if w.MemUsedMax > 0 {
+				break l
+			}
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	st := m.WorkerStats()
+	require.Len(t, st, 1)
+	for _, w := range st {
+		require.Equal(t, storiface.ResourceTable[sealtasks.TTAddPiece][abi.RegisteredSealProof_StackedDrg2KiBV1].MaxMemory, w.MemUsedMax)
+	}
+}
+
+func TestResOverride(t *testing.T) {
+	logging.SetAllLoggers(logging.LevelDebug)
+
+	ctx, done := context.WithCancel(context.Background())
+	defer done()
+
+	ds := datastore.NewMapDatastore()
+
+	m, lstor, stor, idx, cleanup := newTestMgr(ctx, t, ds)
+	defer cleanup()
+
+	localTasks := []sealtasks.TaskType{
+		sealtasks.TTAddPiece, sealtasks.TTFetch,
+	}
+
+	wds := datastore.NewMapDatastore()
+
+	arch := make(chan chan apres)
+	w := newLocalWorker(func() (ffiwrapper.Storage, error) {
+		return &testExec{apch: arch}, nil
+	}, WorkerConfig{
+		TaskTypes: localTasks,
+	}, func(s string) (string, bool) {
+		if s == "AP_2K_MAX_MEMORY" {
+			return "99999", true
+		}
+
+		return "", false
+	}, stor, lstor, idx, m, statestore.New(wds))
+
+	err := m.AddWorker(ctx, w)
+	require.NoError(t, err)
+
+	sid := storage.SectorRef{
+		ID:        abi.SectorID{Miner: 1000, Number: 1},
+		ProofType: abi.RegisteredSealProof_StackedDrg2KiBV1,
+	}
+
+	go func() {
+		_, err := m.AddPiece(ctx, sid, nil, 1016, strings.NewReader(strings.Repeat("testthis", 127)))
+		require.Error(t, err)
+	}()
+
+l:
+	for {
+		st := m.WorkerStats()
+		require.Len(t, st, 1)
+		for _, w := range st {
+			if w.MemUsedMax > 0 {
+				break l
+			}
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	st := m.WorkerStats()
+	require.Len(t, st, 1)
+	for _, w := range st {
+		require.Equal(t, uint64(99999), w.MemUsedMax)
+	}
+}
@@ -12,6 +12,7 @@ import (

 	proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"

+	"github.com/filecoin-project/dagstore/mount"
 	ffiwrapper2 "github.com/filecoin-project/go-commp-utils/ffiwrapper"
 	commcid "github.com/filecoin-project/go-fil-commcid"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -384,12 +385,22 @@ func generateFakePoSt(sectorInfo []proof5.SectorInfo, rpt func(abi.RegisteredSea
 	}
 }

-func (mgr *SectorMgr) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) {
-	if offset != 0 {
+func (mgr *SectorMgr) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (mount.Reader, bool, error) {
+	if uint64(offset) != 0 {
 		panic("implme")
 	}

-	return ioutil.NopCloser(bytes.NewReader(mgr.pieces[mgr.sectors[sector.ID].pieces[0]][:size])), false, nil
+	br := bytes.NewReader(mgr.pieces[mgr.sectors[sector.ID].pieces[0]][:size])
+
+	return struct {
+		io.ReadCloser
+		io.Seeker
+		io.ReaderAt
+	}{
+		ReadCloser: ioutil.NopCloser(br),
+		Seeker:     br,
+		ReaderAt:   br,
+	}, false, nil
 }

 func (mgr *SectorMgr) StageFakeData(mid abi.ActorID, spt abi.RegisteredSealProof) (storage.SectorRef, []abi.PieceInfo, error) {
@@ -8,6 +8,7 @@ import (
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"

+	"github.com/filecoin-project/dagstore/mount"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/specs-storage/storage"

@@ -23,7 +24,11 @@ type Unsealer interface {

 type PieceProvider interface {
 	// ReadPiece is used to read an Unsealed piece at the given offset and of the given size from a Sector
-	ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error)
+	// pieceOffset + pieceSize specify piece bounds for unsealing (note: with SDR the entire sector will be unsealed by
+	//  default in most cases, but this might matter with future PoRep)
+	// startOffset is added to the pieceOffset to get the starting reader offset.
+	// The number of bytes that can be read is pieceSize-startOffset
+	ReadPiece(ctx context.Context, sector storage.SectorRef, pieceOffset storiface.UnpaddedByteIndex, pieceSize abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (mount.Reader, bool, error)
 	IsUnsealed(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error)
 }

@@ -67,50 +72,104 @@ func (p *pieceProvider) IsUnsealed(ctx context.Context, sector storage.SectorRef
 // It will NOT try to schedule an Unseal of a sealed sector file for the read.
 //
 // Returns a nil reader if the piece does NOT exist in any unsealed file or there is no unsealed file for the given sector on any of the workers.
-func (p *pieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (io.ReadCloser, context.CancelFunc, error) {
+func (p *pieceProvider) tryReadUnsealedPiece(ctx context.Context, pc cid.Cid, sector storage.SectorRef, pieceOffset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (mount.Reader, error) {
 	// acquire a lock purely for reading unsealed sectors
 	ctx, cancel := context.WithCancel(ctx)
 	if err := p.index.StorageLock(ctx, sector.ID, storiface.FTUnsealed, storiface.FTNone); err != nil {
 		cancel()
-		return nil, nil, xerrors.Errorf("acquiring read sector lock: %w", err)
+		return nil, xerrors.Errorf("acquiring read sector lock: %w", err)
 	}

-	// Reader returns a reader for an unsealed piece at the given offset in the given sector.
+	// Reader returns a reader getter for an unsealed piece at the given offset in the given sector.
 	// The returned reader will be nil if none of the workers has an unsealed sector file containing
 	// the unsealed piece.
-	r, err := p.storage.Reader(ctx, sector, abi.PaddedPieceSize(offset.Padded()), size.Padded())
+	rg, err := p.storage.Reader(ctx, sector, abi.PaddedPieceSize(pieceOffset.Padded()), size.Padded())
 	if err != nil {
+		cancel()
 		log.Debugf("did not get storage reader;sector=%+v, err:%s", sector.ID, err)
-		cancel()
-		return nil, nil, err
+		return nil, err
 	}
-	if r == nil {
+	if rg == nil {
 		cancel()
+		return nil, nil
 	}

-	return r, cancel, nil
+	buf := make([]byte, fr32.BufSize(size.Padded()))
+
+	pr, err := (&pieceReader{
+		ctx: ctx,
+		getReader: func(ctx context.Context, startOffset uint64) (io.ReadCloser, error) {
+			startOffsetAligned := storiface.UnpaddedByteIndex(startOffset / 127 * 127) // floor to multiple of 127
+
+			r, err := rg(startOffsetAligned.Padded())
+			if err != nil {
+				return nil, xerrors.Errorf("getting reader at +%d: %w", startOffsetAligned, err)
+			}
+
+			upr, err := fr32.NewUnpadReaderBuf(r, size.Padded(), buf)
+			if err != nil {
+				r.Close() // nolint
+				return nil, xerrors.Errorf("creating unpadded reader: %w", err)
+			}
+
+			bir := bufio.NewReaderSize(upr, 127)
+			if startOffset > uint64(startOffsetAligned) {
+				if _, err := bir.Discard(int(startOffset - uint64(startOffsetAligned))); err != nil {
+					r.Close() // nolint
+					return nil, xerrors.Errorf("discarding bytes for startOffset: %w", err)
+				}
+			}
+
+			return struct {
+				io.Reader
+				io.Closer
+			}{
+				Reader: bir,
+				Closer: funcCloser(func() error {
+					return r.Close()
+				}),
+			}, nil
+		},
+		len:      size,
+		onClose:  cancel,
+		pieceCid: pc,
+	}).init()
+	if err != nil || pr == nil { // pr == nil to make sure we don't return typed nil
+		cancel()
+		return nil, err
+	}
+
+	return pr, err
 }

+type funcCloser func() error
+
+func (f funcCloser) Close() error {
+	return f()
+}
+
+var _ io.Closer = funcCloser(nil)
+
 // ReadPiece is used to read an Unsealed piece at the given offset and of the given size from a Sector
 // If an Unsealed sector file exists with the Piece Unsealed in it, we'll use that for the read.
 // Otherwise, we will Unseal a Sealed sector file for the given sector and read the Unsealed piece from it.
 // If we do NOT have an existing unsealed file  containing the given piece thus causing us to schedule an Unseal,
 // the returned boolean parameter will be set to true.
 // If we have an existing unsealed file containing the given piece, the returned boolean will be set to false.
-func (p *pieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) {
-	if err := offset.Valid(); err != nil {
-		return nil, false, xerrors.Errorf("offset is not valid: %w", err)
+func (p *pieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, pieceOffset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (mount.Reader, bool, error) {
+	if err := pieceOffset.Valid(); err != nil {
+		return nil, false, xerrors.Errorf("pieceOffset is not valid: %w", err)
 	}
 	if err := size.Validate(); err != nil {
 		return nil, false, xerrors.Errorf("size is not a valid piece size: %w", err)
 	}

-	r, unlock, err := p.tryReadUnsealedPiece(ctx, sector, offset, size)
+	r, err := p.tryReadUnsealedPiece(ctx, unsealed, sector, pieceOffset, size)

 	log.Debugf("result of first tryReadUnsealedPiece: r=%+v, err=%s", r, err)

 	if xerrors.Is(err, storiface.ErrSectorNotFound) {
-		log.Debugf("no unsealed sector file with unsealed piece, sector=%+v, offset=%d, size=%d", sector, offset, size)
+		log.Debugf("no unsealed sector file with unsealed piece, sector=%+v, pieceOffset=%d, size=%d", sector, pieceOffset, size)
 		err = nil
 	}
 	if err != nil {
@@ -129,14 +188,14 @@ func (p *pieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef,
 		if unsealed == cid.Undef {
 			commd = nil
 		}
-		if err := p.uns.SectorsUnsealPiece(ctx, sector, offset, size, ticket, commd); err != nil {
+		if err := p.uns.SectorsUnsealPiece(ctx, sector, pieceOffset, size, ticket, commd); err != nil {
 			log.Errorf("failed to SectorsUnsealPiece: %s", err)
 			return nil, false, xerrors.Errorf("unsealing piece: %w", err)
 		}

-		log.Debugf("unsealed a sector file to read the piece, sector=%+v, offset=%d, size=%d", sector, offset, size)
+		log.Debugf("unsealed a sector file to read the piece, sector=%+v, pieceOffset=%d, size=%d", sector, pieceOffset, size)

-		r, unlock, err = p.tryReadUnsealedPiece(ctx, sector, offset, size)
+		r, err = p.tryReadUnsealedPiece(ctx, unsealed, sector, pieceOffset, size)
 		if err != nil {
 			log.Errorf("failed to tryReadUnsealedPiece after SectorsUnsealPiece: %s", err)
 			return nil, true, xerrors.Errorf("read after unsealing: %w", err)
@@ -145,32 +204,12 @@ func (p *pieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef,
 			log.Errorf("got no reader after unsealing piece")
 			return nil, true, xerrors.Errorf("got no reader after unsealing piece")
 		}
-		log.Debugf("got a reader to read unsealed piece, sector=%+v, offset=%d, size=%d", sector, offset, size)
+		log.Debugf("got a reader to read unsealed piece, sector=%+v, pieceOffset=%d, size=%d", sector, pieceOffset, size)
 	} else {
-		log.Debugf("unsealed piece already exists, no need to unseal, sector=%+v, offset=%d, size=%d", sector, offset, size)
+		log.Debugf("unsealed piece already exists, no need to unseal, sector=%+v, pieceOffset=%d, size=%d", sector, pieceOffset, size)
 	}

-	upr, err := fr32.NewUnpadReader(r, size.Padded())
-	if err != nil {
-		unlock()
-		return nil, uns, xerrors.Errorf("creating unpadded reader: %w", err)
-	}
+	log.Debugf("returning reader to read unsealed piece, sector=%+v, pieceOffset=%d, size=%d", sector, pieceOffset, size)

-	log.Debugf("returning reader to read unsealed piece, sector=%+v, offset=%d, size=%d", sector, offset, size)
-
-	return &funcCloser{
-		Reader: bufio.NewReaderSize(upr, 127),
-		close: func() error {
-			err = r.Close()
-			unlock()
-			return err
-		},
-	}, uns, nil
+	return r, uns, nil
 }
-
-type funcCloser struct {
-	io.Reader
-	close func() error
-}
-
-func (fc *funcCloser) Close() error { return fc.close() }
@@ -7,6 +7,7 @@ import (
 	"math/rand"
 	"net"
 	"net/http"
+	"os"
 	"testing"

 	"github.com/filecoin-project/go-state-types/abi"
@@ -286,7 +287,7 @@ func (p *pieceProviderTestHarness) addRemoteWorker(t *testing.T, tasks []sealtas

 	worker := newLocalWorker(nil, WorkerConfig{
 		TaskTypes: tasks,
-	}, remote, localStore, p.index, p.mgr, csts)
+	}, os.LookupEnv, remote, localStore, p.index, p.mgr, csts)

 	p.servers = append(p.servers, svc)
 	p.localStores = append(p.localStores, localStore)
@@ -298,7 +299,7 @@ func (p *pieceProviderTestHarness) addRemoteWorker(t *testing.T, tasks []sealtas
 func (p *pieceProviderTestHarness) removeAllUnsealedSectorFiles(t *testing.T) {
 	for i := range p.localStores {
 		ls := p.localStores[i]
-		require.NoError(t, ls.Remove(p.ctx, p.sector.ID, storiface.FTUnsealed, false))
+		require.NoError(t, ls.Remove(p.ctx, p.sector.ID, storiface.FTUnsealed, false, nil))
 	}
 }

@@ -0,0 +1,180 @@
+package sectorstorage
+
+import (
+	"bufio"
+	"context"
+	"io"
+
+	"github.com/ipfs/go-cid"
+	"go.opencensus.io/stats"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/dagstore/mount"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/metrics"
+)
+
+// For small read skips, it's faster to "burn" some bytes than to setup new sector reader.
+// Assuming 1ms stream seek latency, and 1G/s stream rate, we're willing to discard up to 1 MiB.
+var MaxPieceReaderBurnBytes int64 = 1 << 20 // 1M
+var ReadBuf = 128 * (127 * 8)               // unpadded(128k)
+
+type pieceGetter func(ctx context.Context, offset uint64) (io.ReadCloser, error)
+
+type pieceReader struct {
+	ctx       context.Context
+	getReader pieceGetter
+	pieceCid  cid.Cid
+	len       abi.UnpaddedPieceSize
+	onClose   context.CancelFunc
+
+	closed bool
+	seqAt  int64 // next byte to be read by io.Reader
+
+	r   io.ReadCloser
+	br  *bufio.Reader
+	rAt int64
+}
+
+func (p *pieceReader) init() (_ *pieceReader, err error) {
+	stats.Record(p.ctx, metrics.DagStorePRInitCount.M(1))
+
+	p.rAt = 0
+	p.r, err = p.getReader(p.ctx, uint64(p.rAt))
+	if err != nil {
+		return nil, err
+	}
+	if p.r == nil {
+		return nil, nil
+	}
+
+	p.br = bufio.NewReaderSize(p.r, ReadBuf)
+
+	return p, nil
+}
+
+func (p *pieceReader) check() error {
+	if p.closed {
+		return xerrors.Errorf("reader closed")
+	}
+
+	return nil
+}
+
+func (p *pieceReader) Close() error {
+	if err := p.check(); err != nil {
+		return err
+	}
+
+	if p.r != nil {
+		if err := p.r.Close(); err != nil {
+			return err
+		}
+		if err := p.r.Close(); err != nil {
+			return err
+		}
+		p.r = nil
+	}
+
+	p.onClose()
+
+	p.closed = true
+
+	return nil
+}
+
+func (p *pieceReader) Read(b []byte) (int, error) {
+	if err := p.check(); err != nil {
+		return 0, err
+	}
+
+	n, err := p.ReadAt(b, p.seqAt)
+	p.seqAt += int64(n)
+	return n, err
+}
+
+func (p *pieceReader) Seek(offset int64, whence int) (int64, error) {
+	if err := p.check(); err != nil {
+		return 0, err
+	}
+
+	switch whence {
+	case io.SeekStart:
+		p.seqAt = offset
+	case io.SeekCurrent:
+		p.seqAt += offset
+	case io.SeekEnd:
+		p.seqAt = int64(p.len) + offset
+	default:
+		return 0, xerrors.Errorf("bad whence")
+	}
+
+	return p.seqAt, nil
+}
+
+func (p *pieceReader) ReadAt(b []byte, off int64) (n int, err error) {
+	if err := p.check(); err != nil {
+		return 0, err
+	}
+
+	stats.Record(p.ctx, metrics.DagStorePRBytesRequested.M(int64(len(b))))
+
+	// 1. Get the backing reader into the correct position
+
+	// if the backing reader is ahead of the offset we want, or more than
+	//  MaxPieceReaderBurnBytes behind, reset the reader
+	if p.r == nil || p.rAt > off || p.rAt+MaxPieceReaderBurnBytes < off {
+		if p.r != nil {
+			if err := p.r.Close(); err != nil {
+				return 0, xerrors.Errorf("closing backing reader: %w", err)
+			}
+			p.r = nil
+			p.br = nil
+		}
+
+		log.Debugw("pieceReader new stream", "piece", p.pieceCid, "at", p.rAt, "off", off-p.rAt, "n", len(b))
+
+		if off > p.rAt {
+			stats.Record(p.ctx, metrics.DagStorePRSeekForwardBytes.M(off-p.rAt), metrics.DagStorePRSeekForwardCount.M(1))
+		} else {
+			stats.Record(p.ctx, metrics.DagStorePRSeekBackBytes.M(p.rAt-off), metrics.DagStorePRSeekBackCount.M(1))
+		}
+
+		p.rAt = off
+		p.r, err = p.getReader(p.ctx, uint64(p.rAt))
+		p.br = bufio.NewReaderSize(p.r, ReadBuf)
+		if err != nil {
+			return 0, xerrors.Errorf("getting backing reader: %w", err)
+		}
+	}
+
+	// 2. Check if we need to burn some bytes
+	if off > p.rAt {
+		stats.Record(p.ctx, metrics.DagStorePRBytesDiscarded.M(off-p.rAt), metrics.DagStorePRDiscardCount.M(1))
+
+		n, err := io.CopyN(io.Discard, p.br, off-p.rAt)
+		p.rAt += n
+		if err != nil {
+			return 0, xerrors.Errorf("discarding read gap: %w", err)
+		}
+	}
+
+	// 3. Sanity check
+	if off != p.rAt {
+		return 0, xerrors.Errorf("bad reader offset; requested %d; at %d", off, p.rAt)
+	}
+
+	// 4. Read!
+	n, err = io.ReadFull(p.br, b)
+	if n < len(b) {
+		log.Debugw("pieceReader short read", "piece", p.pieceCid, "at", p.rAt, "toEnd", int64(p.len)-p.rAt, "n", len(b), "read", n, "err", err)
+	}
+	if err == io.ErrUnexpectedEOF {
+		err = io.EOF
+	}
+
+	p.rAt += int64(n)
+	return n, err
+}
+
+var _ mount.Reader = (*pieceReader)(nil)
@@ -53,7 +53,7 @@ type WorkerSelector interface {

 type scheduler struct {
 	workersLk sync.RWMutex
-	workers   map[WorkerID]*workerHandle
+	workers   map[storiface.WorkerID]*workerHandle

 	schedule       chan *workerRequest
 	windowRequests chan *schedWindowRequest
@@ -95,7 +95,7 @@ type workerHandle struct {
 }

 type schedWindowRequest struct {
-	worker WorkerID
+	worker storiface.WorkerID

 	done chan *schedWindow
 }
@@ -107,17 +107,18 @@ type schedWindow struct {

 type workerDisableReq struct {
 	activeWindows []*schedWindow
-	wid           WorkerID
+	wid           storiface.WorkerID
 	done          func()
 }

 type activeResources struct {
 	memUsedMin uint64
 	memUsedMax uint64
-	gpuUsed    bool
+	gpuUsed    float64
 	cpuUse     uint64

-	cond *sync.Cond
+	cond    *sync.Cond
+	waiting int
 }

 type workerRequest struct {
@@ -144,7 +145,7 @@ type workerResponse struct {

 func newScheduler() *scheduler {
 	return &scheduler{
-		workers: map[WorkerID]*workerHandle{},
+		workers: map[storiface.WorkerID]*workerHandle{},

 		schedule:       make(chan *workerRequest),
 		windowRequests: make(chan *schedWindowRequest, 20),
@@ -154,8 +155,9 @@ func newScheduler() *scheduler {
 		schedQueue: &requestQueue{},

 		workTracker: &workTracker{
-			done:    map[storiface.CallID]struct{}{},
-			running: map[storiface.CallID]trackedWork{},
+			done:     map[storiface.CallID]struct{}{},
+			running:  map[storiface.CallID]trackedWork{},
+			prepared: map[uuid.UUID]trackedWork{},
 		},

 		info: make(chan func(interface{})),
@@ -376,7 +378,6 @@ func (sh *scheduler) trySched() {
 			}()

 			task := (*sh.schedQueue)[sqi]
-			needRes := ResourceTable[task.taskType][task.sector.ProofType]

 			task.indexHeap = sqi
 			for wnd, windowRequest := range sh.openWindows {
@@ -392,6 +393,8 @@ func (sh *scheduler) trySched() {
 					continue
 				}

+				needRes := worker.info.Resources.ResourceSpec(task.sector.ProofType, task.taskType)
+
 				// TODO: allow bigger windows
 				if !windows[wnd].allocated.canHandleRequest(needRes, windowRequest.worker, "schedAcceptable", worker.info) {
 					continue
@@ -455,7 +458,6 @@ func (sh *scheduler) trySched() {

 	for sqi := 0; sqi < queueLen; sqi++ {
 		task := (*sh.schedQueue)[sqi]
-		needRes := ResourceTable[task.taskType][task.sector.ProofType]

 		selectedWindow := -1
 		for _, wnd := range acceptableWindows[task.indexHeap] {
@@ -464,6 +466,8 @@ func (sh *scheduler) trySched() {

 			log.Debugf("SCHED try assign sqi:%d sector %d to window %d", sqi, task.sector.ID.Number, wnd)

+			needRes := info.Resources.ResourceSpec(task.sector.ProofType, task.taskType)
+
 			// TODO: allow bigger windows
 			if !windows[wnd].allocated.canHandleRequest(needRes, wid, "schedAssign", info) {
 				continue
@@ -6,12 +6,14 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 )

-func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerInfo, r Resources, locker sync.Locker, cb func() error) error {
+func (a *activeResources) withResources(id storiface.WorkerID, wr storiface.WorkerInfo, r storiface.Resources, locker sync.Locker, cb func() error) error {
 	for !a.canHandleRequest(r, id, "withResources", wr) {
 		if a.cond == nil {
 			a.cond = sync.NewCond(locker)
 		}
+		a.waiting++
 		a.cond.Wait()
+		a.waiting--
 	}

 	a.add(wr.Resources, r)
@@ -19,62 +21,79 @@ func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerInfo, r
 	err := cb()

 	a.free(wr.Resources, r)
-	if a.cond != nil {
-		a.cond.Broadcast()
-	}

 	return err
 }

-func (a *activeResources) add(wr storiface.WorkerResources, r Resources) {
-	if r.CanGPU {
-		a.gpuUsed = true
+// must be called with the same lock as the one passed to withResources
+func (a *activeResources) hasWorkWaiting() bool {
+	return a.waiting > 0
+}
+
+func (a *activeResources) add(wr storiface.WorkerResources, r storiface.Resources) {
+	if r.GPUUtilization > 0 {
+		a.gpuUsed += r.GPUUtilization
 	}
-	a.cpuUse += r.Threads(wr.CPUs)
+	a.cpuUse += r.Threads(wr.CPUs, len(wr.GPUs))
 	a.memUsedMin += r.MinMemory
 	a.memUsedMax += r.MaxMemory
 }

-func (a *activeResources) free(wr storiface.WorkerResources, r Resources) {
-	if r.CanGPU {
-		a.gpuUsed = false
+func (a *activeResources) free(wr storiface.WorkerResources, r storiface.Resources) {
+	if r.GPUUtilization > 0 {
+		a.gpuUsed -= r.GPUUtilization
 	}
-	a.cpuUse -= r.Threads(wr.CPUs)
+	a.cpuUse -= r.Threads(wr.CPUs, len(wr.GPUs))
 	a.memUsedMin -= r.MinMemory
 	a.memUsedMax -= r.MaxMemory
+
+	if a.cond != nil {
+		a.cond.Broadcast()
+	}
 }

 // canHandleRequest evaluates if the worker has enough available resources to
 // handle the request.
-func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, caller string, info storiface.WorkerInfo) bool {
+func (a *activeResources) canHandleRequest(needRes storiface.Resources, wid storiface.WorkerID, caller string, info storiface.WorkerInfo) bool {
 	if info.IgnoreResources {
 		// shortcircuit; if this worker is ignoring resources, it can always handle the request.
 		return true
 	}

 	res := info.Resources
+
 	// TODO: dedupe needRes.BaseMinMemory per task type (don't add if that task is already running)
-	minNeedMem := res.MemReserved + a.memUsedMin + needRes.MinMemory + needRes.BaseMinMemory
-	if minNeedMem > res.MemPhysical {
-		log.Debugf("sched: not scheduling on worker %s for %s; not enough physical memory - need: %dM, have %dM", wid, caller, minNeedMem/mib, res.MemPhysical/mib)
+	memNeeded := needRes.MinMemory + needRes.BaseMinMemory
+	memUsed := a.memUsedMin
+	// assume that MemUsed can be swapped, so only check it in the vmem Check
+	memAvail := res.MemPhysical - memUsed
+	if memNeeded > memAvail {
+		log.Debugf("sched: not scheduling on worker %s for %s; not enough physical memory - need: %dM, have %dM available", wid, caller, memNeeded/mib, memAvail/mib)
 		return false
 	}

-	maxNeedMem := res.MemReserved + a.memUsedMax + needRes.MaxMemory + needRes.BaseMinMemory
+	vmemNeeded := needRes.MaxMemory + needRes.BaseMinMemory
+	vmemUsed := a.memUsedMax
+	workerMemoryReserved := res.MemUsed + res.MemSwapUsed // memory used outside lotus-worker (used by the OS, etc.)

-	if maxNeedMem > res.MemSwap+res.MemPhysical {
-		log.Debugf("sched: not scheduling on worker %s for %s; not enough virtual memory - need: %dM, have %dM", wid, caller, maxNeedMem/mib, (res.MemSwap+res.MemPhysical)/mib)
+	if vmemUsed < workerMemoryReserved {
+		vmemUsed = workerMemoryReserved
+	}
+	vmemAvail := (res.MemPhysical + res.MemSwap) - vmemUsed
+
+	if vmemNeeded > vmemAvail {
+		log.Debugf("sched: not scheduling on worker %s for %s; not enough virtual memory - need: %dM, have %dM available", wid, caller, vmemNeeded/mib, vmemAvail/mib)
 		return false
 	}

-	if a.cpuUse+needRes.Threads(res.CPUs) > res.CPUs {
-		log.Debugf("sched: not scheduling on worker %s for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads(res.CPUs), a.cpuUse, res.CPUs)
+	if a.cpuUse+needRes.Threads(res.CPUs, len(res.GPUs)) > res.CPUs {
+		log.Debugf("sched: not scheduling on worker %s for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads(res.CPUs, len(res.GPUs)), a.cpuUse, res.CPUs)
 		return false
 	}

-	if len(res.GPUs) > 0 && needRes.CanGPU {
-		if a.gpuUsed {
-			log.Debugf("sched: not scheduling on worker %s for %s; GPU in use", wid, caller)
+	if len(res.GPUs) > 0 && needRes.GPUUtilization > 0 {
+		if a.gpuUsed+needRes.GPUUtilization > float64(len(res.GPUs)) {
+			log.Debugf("sched: not scheduling on worker %s for %s; GPU(s) in use", wid, caller)
 			return false
 		}
 	}
@@ -88,12 +107,21 @@ func (a *activeResources) utilization(wr storiface.WorkerResources) float64 {
 	cpu := float64(a.cpuUse) / float64(wr.CPUs)
 	max = cpu

-	memMin := float64(a.memUsedMin+wr.MemReserved) / float64(wr.MemPhysical)
+	memUsed := a.memUsedMin
+	if memUsed < wr.MemUsed {
+		memUsed = wr.MemUsed
+	}
+	memMin := float64(memUsed) / float64(wr.MemPhysical)
 	if memMin > max {
 		max = memMin
 	}

-	memMax := float64(a.memUsedMax+wr.MemReserved) / float64(wr.MemPhysical+wr.MemSwap)
+	vmemUsed := a.memUsedMax
+	if a.memUsedMax < wr.MemUsed+wr.MemSwapUsed {
+		vmemUsed = wr.MemUsed + wr.MemSwapUsed
+	}
+	memMax := float64(vmemUsed) / float64(wr.MemPhysical+wr.MemSwap)
+
 	if memMax > max {
 		max = memMax
 	}
@@ -41,14 +41,16 @@ func TestWithPriority(t *testing.T) {
 var decentWorkerResources = storiface.WorkerResources{
 	MemPhysical: 128 << 30,
 	MemSwap:     200 << 30,
-	MemReserved: 2 << 30,
+	MemUsed:     1 << 30,
+	MemSwapUsed: 1 << 30,
 	CPUs:        32,
-	GPUs:        []string{"a GPU"},
+	GPUs:        []string{},
 }

 var constrainedWorkerResources = storiface.WorkerResources{
 	MemPhysical: 1 << 30,
-	MemReserved: 2 << 30,
+	MemUsed:     1 << 30,
+	MemSwapUsed: 1 << 30,
 	CPUs:        1,
 }

@@ -188,6 +190,9 @@ func TestSchedStartStop(t *testing.T) {
 }

 func TestSched(t *testing.T) {
+	storiface.ParallelNum = 1
+	storiface.ParallelDenom = 1
+
 	ctx, done := context.WithTimeout(context.Background(), 30*time.Second)
 	defer done()

@@ -254,7 +259,9 @@ func TestSched(t *testing.T) {

 					return nil
 				}, noopAction)
-				require.NoError(t, err, fmt.Sprint(l, l2))
+				if err != context.Canceled {
+					require.NoError(t, err, fmt.Sprint(l, l2))
+				}
 			}()

 			<-sched.testSync
@@ -299,9 +306,6 @@ func TestSched(t *testing.T) {
 	}

 	testFunc := func(workers []workerSpec, tasks []task) func(t *testing.T) {
-		ParallelNum = 1
-		ParallelDenom = 1
-
 		return func(t *testing.T) {
 			index := stores.NewIndex()

@@ -558,7 +562,7 @@ func BenchmarkTrySched(b *testing.B) {
 				b.StopTimer()

 				sched := newScheduler()
-				sched.workers[WorkerID{}] = &workerHandle{
+				sched.workers[storiface.WorkerID{}] = &workerHandle{
 					workerRpc: nil,
 					info: storiface.WorkerInfo{
 						Hostname:  "t",
@@ -570,7 +574,7 @@ func BenchmarkTrySched(b *testing.B) {

 				for i := 0; i < windows; i++ {
 					sched.openWindows = append(sched.openWindows, &schedWindowRequest{
-						worker: WorkerID{},
+						worker: storiface.WorkerID{},
 						done:   make(chan *schedWindow, 1000),
 					})
 				}
@@ -616,7 +620,7 @@ func TestWindowCompact(t *testing.T) {
 						taskType: task,
 						sector:   storage.SectorRef{ProofType: spt},
 					})
-					window.allocated.add(wh.info.Resources, ResourceTable[task][spt])
+					window.allocated.add(wh.info.Resources, storiface.ResourceTable[task][spt])
 				}

 				wh.activeWindows = append(wh.activeWindows, window)
@@ -635,7 +639,7 @@ func TestWindowCompact(t *testing.T) {

 				for ti, task := range tasks {
 					require.Equal(t, task, wh.activeWindows[wi].todo[ti].taskType, "%d, %d", wi, ti)
-					expectRes.add(wh.info.Resources, ResourceTable[task][spt])
+					expectRes.add(wh.info.Resources, storiface.ResourceTable[task][spt])
 				}

 				require.Equal(t, expectRes.cpuUse, wh.activeWindows[wi].allocated.cpuUse, "%d", wi)
@@ -6,14 +6,16 @@ import (

 	"golang.org/x/xerrors"

+	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 )

 type schedWorker struct {
 	sched  *scheduler
 	worker *workerHandle

-	wid WorkerID
+	wid storiface.WorkerID

 	heartbeatTimer   *time.Ticker
 	scheduledWindows chan *schedWindow
@@ -49,7 +51,7 @@ func (sh *scheduler) runWorker(ctx context.Context, w Worker) error {
 		closedMgr:  make(chan struct{}),
 	}

-	wid := WorkerID(sessID)
+	wid := storiface.WorkerID(sessID)

 	sh.workersLk.Lock()
 	_, exist := sh.workers[wid]
@@ -236,7 +238,7 @@ func (sw *schedWorker) checkSession(ctx context.Context) bool {
 			continue
 		}

-		if WorkerID(curSes) != sw.wid {
+		if storiface.WorkerID(curSes) != sw.wid {
 			if curSes != ClosedWorkerID {
 				// worker restarted
 				log.Warnw("worker session changed (worker restarted?)", "initial", sw.wid, "current", curSes)
@@ -295,7 +297,7 @@ func (sw *schedWorker) workerCompactWindows() {
 			var moved []int

 			for ti, todo := range window.todo {
-				needRes := ResourceTable[todo.taskType][todo.sector.ProofType]
+				needRes := worker.info.Resources.ResourceSpec(todo.sector.ProofType, todo.taskType)
 				if !lower.allocated.canHandleRequest(needRes, sw.wid, "compactWindows", worker.info) {
 					continue
 				}
@@ -338,6 +340,11 @@ func (sw *schedWorker) workerCompactWindows() {
 }

 func (sw *schedWorker) processAssignedWindows() {
+	sw.assignReadyWork()
+	sw.assignPreparingWork()
+}
+
+func (sw *schedWorker) assignPreparingWork() {
 	worker := sw.worker

 assignLoop:
@@ -351,7 +358,7 @@ assignLoop:

 			worker.lk.Lock()
 			for t, todo := range firstWindow.todo {
-				needRes := ResourceTable[todo.taskType][todo.sector.ProofType]
+				needRes := worker.info.Resources.ResourceSpec(todo.sector.ProofType, todo.taskType)
 				if worker.preparing.canHandleRequest(needRes, sw.wid, "startPreparing", worker.info) {
 					tidx = t
 					break
@@ -366,7 +373,7 @@ assignLoop:
 			todo := firstWindow.todo[tidx]

 			log.Debugf("assign worker sector %d", todo.sector.ID.Number)
-			err := sw.startProcessingTask(sw.taskDone, todo)
+			err := sw.startProcessingTask(todo)

 			if err != nil {
 				log.Errorf("startProcessingTask error: %+v", err)
@@ -387,10 +394,70 @@ assignLoop:
 	}
 }

-func (sw *schedWorker) startProcessingTask(taskDone chan struct{}, req *workerRequest) error {
+func (sw *schedWorker) assignReadyWork() {
+	worker := sw.worker
+
+	worker.lk.Lock()
+	defer worker.lk.Unlock()
+
+	if worker.active.hasWorkWaiting() {
+		// prepared tasks have priority
+		return
+	}
+
+assignLoop:
+	// process windows in order
+	for len(worker.activeWindows) > 0 {
+		firstWindow := worker.activeWindows[0]
+
+		// process tasks within a window, preferring tasks at lower indexes
+		for len(firstWindow.todo) > 0 {
+			tidx := -1
+
+			for t, todo := range firstWindow.todo {
+				if todo.taskType != sealtasks.TTCommit1 && todo.taskType != sealtasks.TTCommit2 { // todo put in task
+					continue
+				}
+
+				needRes := storiface.ResourceTable[todo.taskType][todo.sector.ProofType]
+				if worker.active.canHandleRequest(needRes, sw.wid, "startPreparing", worker.info) {
+					tidx = t
+					break
+				}
+			}
+
+			if tidx == -1 {
+				break assignLoop
+			}
+
+			todo := firstWindow.todo[tidx]
+
+			log.Debugf("assign worker sector %d (ready)", todo.sector.ID.Number)
+			err := sw.startProcessingReadyTask(todo)
+
+			if err != nil {
+				log.Errorf("startProcessingTask error: %+v", err)
+				go todo.respond(xerrors.Errorf("startProcessingTask error: %w", err))
+			}
+
+			// Note: we're not freeing window.allocated resources here very much on purpose
+			copy(firstWindow.todo[tidx:], firstWindow.todo[tidx+1:])
+			firstWindow.todo[len(firstWindow.todo)-1] = nil
+			firstWindow.todo = firstWindow.todo[:len(firstWindow.todo)-1]
+		}
+
+		copy(worker.activeWindows, worker.activeWindows[1:])
+		worker.activeWindows[len(worker.activeWindows)-1] = nil
+		worker.activeWindows = worker.activeWindows[:len(worker.activeWindows)-1]
+
+		sw.windowsRequested--
+	}
+}
+
+func (sw *schedWorker) startProcessingTask(req *workerRequest) error {
 	w, sh := sw.worker, sw.sched

-	needRes := ResourceTable[req.taskType][req.sector.ProofType]
+	needRes := w.info.Resources.ResourceSpec(req.sector.ProofType, req.taskType)

 	w.lk.Lock()
 	w.preparing.add(w.info.Resources, needRes)
@@ -398,7 +465,9 @@ func (sw *schedWorker) startProcessingTask(taskDone chan struct{}, req *workerRe

 	go func() {
 		// first run the prepare step (e.g. fetching sector data from other worker)
-		err := req.prepare(req.ctx, sh.workTracker.worker(sw.wid, w.info, w.workerRpc))
+		tw := sh.workTracker.worker(sw.wid, w.info, w.workerRpc)
+		tw.start()
+		err := req.prepare(req.ctx, tw)
 		w.lk.Lock()

 		if err != nil {
@@ -406,9 +475,10 @@ func (sw *schedWorker) startProcessingTask(taskDone chan struct{}, req *workerRe
 			w.lk.Unlock()

 			select {
-			case taskDone <- struct{}{}:
+			case sw.taskDone <- struct{}{}:
 			case <-sh.closing:
 				log.Warnf("scheduler closed while sending response (prepare error: %+v)", err)
+			default: // there is a notification pending already
 			}

 			select {
@@ -421,6 +491,14 @@ func (sw *schedWorker) startProcessingTask(taskDone chan struct{}, req *workerRe
 			return
 		}

+		tw = sh.workTracker.worker(sw.wid, w.info, w.workerRpc)
+
+		// start tracking work first early in case we need to wait for resources
+		werr := make(chan error, 1)
+		go func() {
+			werr <- req.work(req.ctx, tw)
+		}()
+
 		// wait (if needed) for resources in the 'active' window
 		err = w.active.withResources(sw.wid, w.info, needRes, &w.lk, func() error {
 			w.preparing.free(w.info.Resources, needRes)
@@ -428,12 +506,14 @@ func (sw *schedWorker) startProcessingTask(taskDone chan struct{}, req *workerRe
 			defer w.lk.Lock() // we MUST return locked from this function

 			select {
-			case taskDone <- struct{}{}:
+			case sw.taskDone <- struct{}{}:
 			case <-sh.closing:
+			default: // there is a notification pending already
 			}

 			// Do the work!
-			err = req.work(req.ctx, sh.workTracker.worker(sw.wid, w.info, w.workerRpc))
+			tw.start()
+			err = <-werr

 			select {
 			case req.ret <- workerResponse{err: err}:
@@ -457,7 +537,50 @@ func (sw *schedWorker) startProcessingTask(taskDone chan struct{}, req *workerRe
 	return nil
 }

-func (sh *scheduler) workerCleanup(wid WorkerID, w *workerHandle) {
+func (sw *schedWorker) startProcessingReadyTask(req *workerRequest) error {
+	w, sh := sw.worker, sw.sched
+
+	needRes := w.info.Resources.ResourceSpec(req.sector.ProofType, req.taskType)
+
+	w.active.add(w.info.Resources, needRes)
+
+	go func() {
+		// Do the work!
+		tw := sh.workTracker.worker(sw.wid, w.info, w.workerRpc)
+		tw.start()
+		err := req.work(req.ctx, tw)
+
+		select {
+		case req.ret <- workerResponse{err: err}:
+		case <-req.ctx.Done():
+			log.Warnf("request got cancelled before we could respond")
+		case <-sh.closing:
+			log.Warnf("scheduler closed while sending response")
+		}
+
+		w.lk.Lock()
+
+		w.active.free(w.info.Resources, needRes)
+
+		select {
+		case sw.taskDone <- struct{}{}:
+		case <-sh.closing:
+			log.Warnf("scheduler closed while sending response (prepare error: %+v)", err)
+		default: // there is a notification pending already
+		}
+
+		w.lk.Unlock()
+
+		// This error should always be nil, since nothing is setting it, but just to be safe:
+		if err != nil {
+			log.Errorf("error executing worker (ready): %+v", err)
+		}
+	}()
+
+	return nil
+}
+
+func (sh *scheduler) workerCleanup(wid storiface.WorkerID, w *workerHandle) {
 	select {
 	case <-w.closingMgr:
 	default:
@@ -35,7 +35,13 @@ func (m *Manager) WorkerJobs() map[uuid.UUID][]storiface.WorkerJob {
 	out := map[uuid.UUID][]storiface.WorkerJob{}
 	calls := map[storiface.CallID]struct{}{}

-	for _, t := range m.sched.workTracker.Running() {
+	running, preparing := m.sched.workTracker.Running()
+
+	for _, t := range running {
+		out[uuid.UUID(t.worker)] = append(out[uuid.UUID(t.worker)], t.job)
+		calls[t.job.ID] = struct{}{}
+	}
+	for _, t := range preparing {
 		out[uuid.UUID(t.worker)] = append(out[uuid.UUID(t.worker)], t.job)
 		calls[t.job.ID] = struct{}{}
 	}
@@ -50,7 +56,7 @@ func (m *Manager) WorkerJobs() map[uuid.UUID][]storiface.WorkerJob {
 					ID:      storiface.UndefCall,
 					Sector:  request.sector.ID,
 					Task:    request.taskType,
-					RunWait: wi + 1,
+					RunWait: wi + 2,
 					Start:   request.start,
 				})
 			}
@@ -2,7 +2,6 @@ package stores

 import (
 	"encoding/json"
-	"io"
 	"net/http"
 	"os"
 	"strconv"
@@ -21,9 +20,9 @@ import (

 var log = logging.Logger("stores")

-var _ partialFileHandler = &DefaultPartialFileHandler{}
+var _ PartialFileHandler = &DefaultPartialFileHandler{}

-// DefaultPartialFileHandler is the default implementation of the partialFileHandler interface.
+// DefaultPartialFileHandler is the default implementation of the PartialFileHandler interface.
 // This is probably the only implementation we'll ever use because the purpose of the
 // interface to is to mock out partial file related functionality during testing.
 type DefaultPartialFileHandler struct{}
@@ -46,7 +45,7 @@ func (d *DefaultPartialFileHandler) Close(pf *partialfile.PartialFile) error {

 type FetchHandler struct {
 	Local     Store
-	PfHandler partialFileHandler
+	PfHandler PartialFileHandler
 }

 func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /remote/
@@ -85,7 +84,6 @@ func (handler *FetchHandler) remoteStatFs(w http.ResponseWriter, r *http.Request
 // remoteGetSector returns the sector file/tared directory byte stream for the sectorID and sector file type sent in the request.
 // returns an error if it does NOT have the required sector file/dir.
 func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Request) {
-	log.Infof("SERVE GET %s", r.URL)
 	vars := mux.Vars(r)

 	id, err := storiface.ParseSectorID(vars["id"])
@@ -139,17 +137,12 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ
 			return
 		}

-		rd, err := tarutil.TarDirectory(path)
-		if err != nil {
-			log.Errorf("%+v", err)
-			w.WriteHeader(500)
-			return
-		}
-
 		w.Header().Set("Content-Type", "application/x-tar")
 		w.WriteHeader(200)
-		if _, err := io.CopyBuffer(w, rd, make([]byte, CopyBuf)); err != nil {
-			log.Errorf("%+v", err)
+
+		err := tarutil.TarDirectory(path, w, make([]byte, CopyBuf))
+		if err != nil {
+			log.Errorf("send tar: %+v", err)
 			return
 		}
 	} else {
@@ -179,7 +172,7 @@ func (handler *FetchHandler) remoteDeleteSector(w http.ResponseWriter, r *http.R
 		return
 	}

-	if err := handler.Local.Remove(r.Context(), id, ft, false); err != nil {
+	if err := handler.Local.Remove(r.Context(), id, ft, false, []ID{ID(r.FormValue("keep"))}); err != nil {
 		log.Errorf("%+v", err)
 		w.WriteHeader(500)
 		return
@@ -63,7 +63,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 	tcs := map[string]struct {
 		piFnc    func(pi *pieceInfo)
 		storeFnc func(s *mocks.MockStore)
-		pfFunc   func(s *mocks.MockpartialFileHandler)
+		pfFunc   func(s *mocks.MockPartialFileHandler)

 		// expectation
 		expectedStatusCode int
@@ -129,7 +129,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 					storiface.SectorPaths{}, nil).Times(1)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(&partialfile.PartialFile{},
 					xerrors.New("some error")).Times(1)
 			},
@@ -146,7 +146,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 					storiface.SectorPaths{}, nil).Times(1)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(emptyPartialFile,
 					nil).Times(1)

@@ -165,7 +165,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 					storiface.SectorPaths{}, nil).Times(1)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(emptyPartialFile,
 					nil).Times(1)

@@ -184,7 +184,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 					storiface.SectorPaths{}, nil).Times(1)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(emptyPartialFile,
 					nil).Times(1)

@@ -203,7 +203,7 @@ func TestRemoteGetAllocated(t *testing.T) {
 			defer mockCtrl.Finish()

 			lstore := mocks.NewMockStore(mockCtrl)
-			pfhandler := mocks.NewMockpartialFileHandler(mockCtrl)
+			pfhandler := mocks.NewMockPartialFileHandler(mockCtrl)

 			handler := &stores.FetchHandler{
 				lstore,
@@ -371,7 +371,7 @@ func TestRemoteGetSector(t *testing.T) {
 			// when test is done, assert expectations on all mock objects.
 			defer mockCtrl.Finish()
 			lstore := mocks.NewMockStore(mockCtrl)
-			pfhandler := mocks.NewMockpartialFileHandler(mockCtrl)
+			pfhandler := mocks.NewMockPartialFileHandler(mockCtrl)

 			var path string

@@ -29,6 +29,8 @@ var SkippedHeartbeatThresh = HeartbeatInterval * 5
 //  filesystem, local or networked / shared by multiple machines
 type ID string

+type Group = string
+
 type StorageInfo struct {
 	ID         ID
 	URLs       []string // TODO: Support non-http transports
@@ -37,6 +39,9 @@ type StorageInfo struct {

 	CanSeal  bool
 	CanStore bool
+
+	Groups  []Group
+	AllowTo []Group
 }

 type HealthReport struct {
@@ -55,6 +60,8 @@ type SectorStorageInfo struct {
 	Primary bool
 }

+//go:generate go run github.com/golang/mock/mockgen -destination=mocks/index.go -package=mocks . SectorIndex
+
 type SectorIndex interface { // part of storage-miner api
 	StorageAttach(context.Context, StorageInfo, fsutil.FsStat) error
 	StorageInfo(context.Context, ID) (StorageInfo, error)
@@ -166,6 +173,8 @@ func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st fsutil.FsS
 		i.stores[si.ID].info.MaxStorage = si.MaxStorage
 		i.stores[si.ID].info.CanSeal = si.CanSeal
 		i.stores[si.ID].info.CanStore = si.CanStore
+		i.stores[si.ID].info.Groups = si.Groups
+		i.stores[si.ID].info.AllowTo = si.AllowTo

 		return nil
 	}
@@ -290,6 +299,8 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft storif
 	storageIDs := map[ID]uint64{}
 	isprimary := map[ID]bool{}

+	allowTo := map[Group]struct{}{}
+
 	for _, pathType := range storiface.PathTypes {
 		if ft&pathType == 0 {
 			continue
@@ -321,6 +332,14 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft storif
 			urls[k] = rl.String()
 		}

+		if allowTo != nil && len(st.info.AllowTo) > 0 {
+			for _, group := range st.info.AllowTo {
+				allowTo[group] = struct{}{}
+			}
+		} else {
+			allowTo = nil // allow to any
+		}
+
 		out = append(out, SectorStorageInfo{
 			ID:     id,
 			URLs:   urls,
@@ -363,6 +382,22 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft storif
 				continue
 			}

+			if allowTo != nil {
+				allow := false
+				for _, group := range st.info.Groups {
+					if _, found := allowTo[group]; found {
+						log.Debugf("path %s in allowed group %s", st.info.ID, group)
+						allow = true
+						break
+					}
+				}
+
+				if !allow {
+					log.Debugf("not selecting on %s, not in allowed group, allow %+v; path has %+v", st.info.ID, allowTo, st.info.Groups)
+					continue
+				}
+			}
+
 			urls := make([]string, len(st.info.URLs))
 			for k, u := range st.info.URLs {
 				rl, err := url.Parse(u)
@@ -0,0 +1,154 @@
+package stores
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/uuid"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+)
+
+func init() {
+	logging.SetLogLevel("stores", "DEBUG")
+}
+
+func newTestStorage() StorageInfo {
+	return StorageInfo{
+		ID:       ID(uuid.New().String()),
+		CanSeal:  true,
+		CanStore: true,
+		Groups:   nil,
+		AllowTo:  nil,
+	}
+}
+
+var bigFsStat = fsutil.FsStat{
+	Capacity:    1 << 40,
+	Available:   1 << 40,
+	FSAvailable: 1 << 40,
+	Reserved:    0,
+	Max:         0,
+	Used:        0,
+}
+
+const s32g = 32 << 30
+
+func TestFindSimple(t *testing.T) {
+	ctx := context.Background()
+
+	i := NewIndex()
+	stor1 := newTestStorage()
+	stor2 := newTestStorage()
+
+	require.NoError(t, i.StorageAttach(ctx, stor1, bigFsStat))
+	require.NoError(t, i.StorageAttach(ctx, stor2, bigFsStat))
+
+	s1 := abi.SectorID{
+		Miner:  12,
+		Number: 34,
+	}
+
+	{
+		si, err := i.StorageFindSector(ctx, s1, storiface.FTSealed, s32g, true)
+		require.NoError(t, err)
+		require.Len(t, si, 0)
+	}
+
+	require.NoError(t, i.StorageDeclareSector(ctx, stor1.ID, s1, storiface.FTSealed, true))
+
+	{
+		si, err := i.StorageFindSector(ctx, s1, storiface.FTSealed, s32g, false)
+		require.NoError(t, err)
+		require.Len(t, si, 1)
+		require.Equal(t, stor1.ID, si[0].ID)
+	}
+
+	{
+		si, err := i.StorageFindSector(ctx, s1, storiface.FTSealed, s32g, true)
+		require.NoError(t, err)
+		require.Len(t, si, 2)
+	}
+}
+
+func TestFindNoAllow(t *testing.T) {
+	ctx := context.Background()
+
+	i := NewIndex()
+	stor1 := newTestStorage()
+	stor1.AllowTo = []Group{"grp1"}
+	stor2 := newTestStorage()
+
+	require.NoError(t, i.StorageAttach(ctx, stor1, bigFsStat))
+	require.NoError(t, i.StorageAttach(ctx, stor2, bigFsStat))
+
+	s1 := abi.SectorID{
+		Miner:  12,
+		Number: 34,
+	}
+	require.NoError(t, i.StorageDeclareSector(ctx, stor1.ID, s1, storiface.FTSealed, true))
+
+	{
+		si, err := i.StorageFindSector(ctx, s1, storiface.FTSealed, s32g, false)
+		require.NoError(t, err)
+		require.Len(t, si, 1)
+		require.Equal(t, stor1.ID, si[0].ID)
+	}
+
+	{
+		si, err := i.StorageFindSector(ctx, s1, storiface.FTSealed, s32g, true)
+		require.NoError(t, err)
+		require.Len(t, si, 1)
+		require.Equal(t, stor1.ID, si[0].ID)
+	}
+}
+
+func TestFindAllow(t *testing.T) {
+	ctx := context.Background()
+
+	i := NewIndex()
+
+	stor1 := newTestStorage()
+	stor1.AllowTo = []Group{"grp1"}
+
+	stor2 := newTestStorage()
+	stor2.Groups = []Group{"grp1"}
+
+	stor3 := newTestStorage()
+	stor3.Groups = []Group{"grp2"}
+
+	require.NoError(t, i.StorageAttach(ctx, stor1, bigFsStat))
+	require.NoError(t, i.StorageAttach(ctx, stor2, bigFsStat))
+	require.NoError(t, i.StorageAttach(ctx, stor3, bigFsStat))
+
+	s1 := abi.SectorID{
+		Miner:  12,
+		Number: 34,
+	}
+	require.NoError(t, i.StorageDeclareSector(ctx, stor1.ID, s1, storiface.FTSealed, true))
+
+	{
+		si, err := i.StorageFindSector(ctx, s1, storiface.FTSealed, s32g, false)
+		require.NoError(t, err)
+		require.Len(t, si, 1)
+		require.Equal(t, stor1.ID, si[0].ID)
+	}
+
+	{
+		si, err := i.StorageFindSector(ctx, s1, storiface.FTSealed, s32g, true)
+		require.NoError(t, err)
+		require.Len(t, si, 2)
+		if si[0].ID == stor1.ID {
+			require.Equal(t, stor1.ID, si[0].ID)
+			require.Equal(t, stor2.ID, si[1].ID)
+		} else {
+			require.Equal(t, stor1.ID, si[1].ID)
+			require.Equal(t, stor2.ID, si[0].ID)
+		}
+	}
+}
@@ -13,8 +13,10 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 )

+//go:generate go run github.com/golang/mock/mockgen -destination=mocks/pf.go -package=mocks . PartialFileHandler
+
 // PartialFileHandler helps mock out the partial file functionality during testing.
-type partialFileHandler interface {
+type PartialFileHandler interface {
 	// OpenPartialFile opens and returns a partial file at the given path and also verifies it has the given
 	// size
 	OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialfile.PartialFile, error)
@@ -30,9 +32,11 @@ type partialFileHandler interface {
 	Close(pf *partialfile.PartialFile) error
 }

+//go:generate go run github.com/golang/mock/mockgen -destination=mocks/store.go -package=mocks . Store
+
 type Store interface {
 	AcquireSector(ctx context.Context, s storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, sealing storiface.PathType, op storiface.AcquireMode) (paths storiface.SectorPaths, stores storiface.SectorPaths, err error)
-	Remove(ctx context.Context, s abi.SectorID, types storiface.SectorFileType, force bool) error
+	Remove(ctx context.Context, s abi.SectorID, types storiface.SectorFileType, force bool, keepIn []ID) error

 	// like remove, but doesn't remove the primary sector copy, nor the last
 	// non-primary copy if there no primary copies
@@ -46,6 +46,13 @@ type LocalStorageMeta struct {
 	// MaxStorage specifies the maximum number of bytes to use for sector storage
 	// (0 = unlimited)
 	MaxStorage uint64
+
+	// List of storage groups this path belongs to
+	Groups []string
+
+	// List of storage groups to which data from this path can be moved. If none
+	// are specified, allow to all
+	AllowTo []string
 }

 // StorageConfig .lotusstorage/storage.json
@@ -212,6 +219,8 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
 		MaxStorage: meta.MaxStorage,
 		CanSeal:    meta.CanSeal,
 		CanStore:   meta.CanStore,
+		Groups:     meta.Groups,
+		AllowTo:    meta.AllowTo,
 	}, fst)
 	if err != nil {
 		return xerrors.Errorf("declaring storage in index: %w", err)
@@ -276,6 +285,8 @@ func (st *Local) Redeclare(ctx context.Context) error {
 			MaxStorage: meta.MaxStorage,
 			CanSeal:    meta.CanSeal,
 			CanStore:   meta.CanStore,
+			Groups:     meta.Groups,
+			AllowTo:    meta.AllowTo,
 		}, fst)
 		if err != nil {
 			return xerrors.Errorf("redeclaring storage in index: %w", err)
@@ -544,7 +555,7 @@ func (st *Local) Local(ctx context.Context) ([]StoragePath, error) {
 	return out, nil
 }

-func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool) error {
+func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool, keepIn []ID) error {
 	if bits.OnesCount(uint(typ)) != 1 {
 		return xerrors.New("delete expects one file type")
 	}
@@ -558,7 +569,14 @@ func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.Sec
 		return xerrors.Errorf("can't delete sector %v(%d), not found", sid, typ)
 	}

+storeLoop:
 	for _, info := range si {
+		for _, id := range keepIn {
+			if id == info.ID {
+				continue storeLoop
+			}
+		}
+
 		if err := st.removeSector(ctx, sid, typ, info.ID); err != nil {
 			return err
 		}
@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: index.go
+// Source: github.com/filecoin-project/lotus/extern/sector-storage/stores (interfaces: SectorIndex)

 // Package mocks is a generated GoMock package.
 package mocks
@@ -53,61 +53,61 @@ func (mr *MockSectorIndexMockRecorder) StorageAttach(arg0, arg1, arg2 interface{
 }

 // StorageBestAlloc mocks base method.
-func (m *MockSectorIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error) {
+func (m *MockSectorIndex) StorageBestAlloc(arg0 context.Context, arg1 storiface.SectorFileType, arg2 abi.SectorSize, arg3 storiface.PathType) ([]stores.StorageInfo, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageBestAlloc", ctx, allocate, ssize, pathType)
+	ret := m.ctrl.Call(m, "StorageBestAlloc", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].([]stores.StorageInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

 // StorageBestAlloc indicates an expected call of StorageBestAlloc.
-func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(ctx, allocate, ssize, pathType interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), ctx, allocate, ssize, pathType)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), arg0, arg1, arg2, arg3)
 }

 // StorageDeclareSector mocks base method.
-func (m *MockSectorIndex) StorageDeclareSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error {
+func (m *MockSectorIndex) StorageDeclareSector(arg0 context.Context, arg1 stores.ID, arg2 abi.SectorID, arg3 storiface.SectorFileType, arg4 bool) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageDeclareSector", ctx, storageID, s, ft, primary)
+	ret := m.ctrl.Call(m, "StorageDeclareSector", arg0, arg1, arg2, arg3, arg4)
 	ret0, _ := ret[0].(error)
 	return ret0
 }

 // StorageDeclareSector indicates an expected call of StorageDeclareSector.
-func (mr *MockSectorIndexMockRecorder) StorageDeclareSector(ctx, storageID, s, ft, primary interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageDeclareSector(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageDeclareSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageDeclareSector), ctx, storageID, s, ft, primary)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageDeclareSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageDeclareSector), arg0, arg1, arg2, arg3, arg4)
 }

 // StorageDropSector mocks base method.
-func (m *MockSectorIndex) StorageDropSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType) error {
+func (m *MockSectorIndex) StorageDropSector(arg0 context.Context, arg1 stores.ID, arg2 abi.SectorID, arg3 storiface.SectorFileType) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageDropSector", ctx, storageID, s, ft)
+	ret := m.ctrl.Call(m, "StorageDropSector", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(error)
 	return ret0
 }

 // StorageDropSector indicates an expected call of StorageDropSector.
-func (mr *MockSectorIndexMockRecorder) StorageDropSector(ctx, storageID, s, ft interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageDropSector(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageDropSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageDropSector), ctx, storageID, s, ft)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageDropSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageDropSector), arg0, arg1, arg2, arg3)
 }

 // StorageFindSector mocks base method.
-func (m *MockSectorIndex) StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) {
+func (m *MockSectorIndex) StorageFindSector(arg0 context.Context, arg1 abi.SectorID, arg2 storiface.SectorFileType, arg3 abi.SectorSize, arg4 bool) ([]stores.SectorStorageInfo, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageFindSector", ctx, sector, ft, ssize, allowFetch)
+	ret := m.ctrl.Call(m, "StorageFindSector", arg0, arg1, arg2, arg3, arg4)
 	ret0, _ := ret[0].([]stores.SectorStorageInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

 // StorageFindSector indicates an expected call of StorageFindSector.
-func (mr *MockSectorIndexMockRecorder) StorageFindSector(ctx, sector, ft, ssize, allowFetch interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageFindSector(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageFindSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageFindSector), ctx, sector, ft, ssize, allowFetch)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageFindSector", reflect.TypeOf((*MockSectorIndex)(nil).StorageFindSector), arg0, arg1, arg2, arg3, arg4)
 }

 // StorageInfo mocks base method.
@@ -126,32 +126,32 @@ func (mr *MockSectorIndexMockRecorder) StorageInfo(arg0, arg1 interface{}) *gomo
 }

 // StorageList mocks base method.
-func (m *MockSectorIndex) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) {
+func (m *MockSectorIndex) StorageList(arg0 context.Context) (map[stores.ID][]stores.Decl, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageList", ctx)
+	ret := m.ctrl.Call(m, "StorageList", arg0)
 	ret0, _ := ret[0].(map[stores.ID][]stores.Decl)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

 // StorageList indicates an expected call of StorageList.
-func (mr *MockSectorIndexMockRecorder) StorageList(ctx interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageList(arg0 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageList", reflect.TypeOf((*MockSectorIndex)(nil).StorageList), ctx)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageList", reflect.TypeOf((*MockSectorIndex)(nil).StorageList), arg0)
 }

 // StorageLock mocks base method.
-func (m *MockSectorIndex) StorageLock(ctx context.Context, sector abi.SectorID, read, write storiface.SectorFileType) error {
+func (m *MockSectorIndex) StorageLock(arg0 context.Context, arg1 abi.SectorID, arg2, arg3 storiface.SectorFileType) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageLock", ctx, sector, read, write)
+	ret := m.ctrl.Call(m, "StorageLock", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(error)
 	return ret0
 }

 // StorageLock indicates an expected call of StorageLock.
-func (mr *MockSectorIndexMockRecorder) StorageLock(ctx, sector, read, write interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageLock(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageLock", reflect.TypeOf((*MockSectorIndex)(nil).StorageLock), ctx, sector, read, write)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageLock", reflect.TypeOf((*MockSectorIndex)(nil).StorageLock), arg0, arg1, arg2, arg3)
 }

 // StorageReportHealth mocks base method.
@@ -169,16 +169,16 @@ func (mr *MockSectorIndexMockRecorder) StorageReportHealth(arg0, arg1, arg2 inte
 }

 // StorageTryLock mocks base method.
-func (m *MockSectorIndex) StorageTryLock(ctx context.Context, sector abi.SectorID, read, write storiface.SectorFileType) (bool, error) {
+func (m *MockSectorIndex) StorageTryLock(arg0 context.Context, arg1 abi.SectorID, arg2, arg3 storiface.SectorFileType) (bool, error) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StorageTryLock", ctx, sector, read, write)
+	ret := m.ctrl.Call(m, "StorageTryLock", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(bool)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

 // StorageTryLock indicates an expected call of StorageTryLock.
-func (mr *MockSectorIndexMockRecorder) StorageTryLock(ctx, sector, read, write interface{}) *gomock.Call {
+func (mr *MockSectorIndexMockRecorder) StorageTryLock(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageTryLock", reflect.TypeOf((*MockSectorIndex)(nil).StorageTryLock), ctx, sector, read, write)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageTryLock", reflect.TypeOf((*MockSectorIndex)(nil).StorageTryLock), arg0, arg1, arg2, arg3)
 }
@@ -0,0 +1,97 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/filecoin-project/lotus/extern/sector-storage/stores (interfaces: PartialFileHandler)
+
+// Package mocks is a generated GoMock package.
+package mocks
+
+import (
+	os "os"
+	reflect "reflect"
+
+	abi "github.com/filecoin-project/go-state-types/abi"
+	partialfile "github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
+	storiface "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockPartialFileHandler is a mock of PartialFileHandler interface.
+type MockPartialFileHandler struct {
+	ctrl     *gomock.Controller
+	recorder *MockPartialFileHandlerMockRecorder
+}
+
+// MockPartialFileHandlerMockRecorder is the mock recorder for MockPartialFileHandler.
+type MockPartialFileHandlerMockRecorder struct {
+	mock *MockPartialFileHandler
+}
+
+// NewMockPartialFileHandler creates a new mock instance.
+func NewMockPartialFileHandler(ctrl *gomock.Controller) *MockPartialFileHandler {
+	mock := &MockPartialFileHandler{ctrl: ctrl}
+	mock.recorder = &MockPartialFileHandlerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockPartialFileHandler) EXPECT() *MockPartialFileHandlerMockRecorder {
+	return m.recorder
+}
+
+// Close mocks base method.
+func (m *MockPartialFileHandler) Close(arg0 *partialfile.PartialFile) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Close", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Close indicates an expected call of Close.
+func (mr *MockPartialFileHandlerMockRecorder) Close(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockPartialFileHandler)(nil).Close), arg0)
+}
+
+// HasAllocated mocks base method.
+func (m *MockPartialFileHandler) HasAllocated(arg0 *partialfile.PartialFile, arg1 storiface.UnpaddedByteIndex, arg2 abi.UnpaddedPieceSize) (bool, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "HasAllocated", arg0, arg1, arg2)
+	ret0, _ := ret[0].(bool)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// HasAllocated indicates an expected call of HasAllocated.
+func (mr *MockPartialFileHandlerMockRecorder) HasAllocated(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasAllocated", reflect.TypeOf((*MockPartialFileHandler)(nil).HasAllocated), arg0, arg1, arg2)
+}
+
+// OpenPartialFile mocks base method.
+func (m *MockPartialFileHandler) OpenPartialFile(arg0 abi.PaddedPieceSize, arg1 string) (*partialfile.PartialFile, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "OpenPartialFile", arg0, arg1)
+	ret0, _ := ret[0].(*partialfile.PartialFile)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// OpenPartialFile indicates an expected call of OpenPartialFile.
+func (mr *MockPartialFileHandlerMockRecorder) OpenPartialFile(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenPartialFile", reflect.TypeOf((*MockPartialFileHandler)(nil).OpenPartialFile), arg0, arg1)
+}
+
+// Reader mocks base method.
+func (m *MockPartialFileHandler) Reader(arg0 *partialfile.PartialFile, arg1 storiface.PaddedByteIndex, arg2 abi.PaddedPieceSize) (*os.File, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Reader", arg0, arg1, arg2)
+	ret0, _ := ret[0].(*os.File)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Reader indicates an expected call of Reader.
+func (mr *MockPartialFileHandlerMockRecorder) Reader(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reader", reflect.TypeOf((*MockPartialFileHandler)(nil).Reader), arg0, arg1, arg2)
+}
@@ -0,0 +1,128 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/filecoin-project/lotus/extern/sector-storage/stores (interfaces: Store)
+
+// Package mocks is a generated GoMock package.
+package mocks
+
+import (
+	context "context"
+	reflect "reflect"
+
+	abi "github.com/filecoin-project/go-state-types/abi"
+	fsutil "github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
+	stores "github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	storiface "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	storage "github.com/filecoin-project/specs-storage/storage"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockStore is a mock of Store interface.
+type MockStore struct {
+	ctrl     *gomock.Controller
+	recorder *MockStoreMockRecorder
+}
+
+// MockStoreMockRecorder is the mock recorder for MockStore.
+type MockStoreMockRecorder struct {
+	mock *MockStore
+}
+
+// NewMockStore creates a new mock instance.
+func NewMockStore(ctrl *gomock.Controller) *MockStore {
+	mock := &MockStore{ctrl: ctrl}
+	mock.recorder = &MockStoreMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockStore) EXPECT() *MockStoreMockRecorder {
+	return m.recorder
+}
+
+// AcquireSector mocks base method.
+func (m *MockStore) AcquireSector(arg0 context.Context, arg1 storage.SectorRef, arg2, arg3 storiface.SectorFileType, arg4 storiface.PathType, arg5 storiface.AcquireMode) (storiface.SectorPaths, storiface.SectorPaths, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "AcquireSector", arg0, arg1, arg2, arg3, arg4, arg5)
+	ret0, _ := ret[0].(storiface.SectorPaths)
+	ret1, _ := ret[1].(storiface.SectorPaths)
+	ret2, _ := ret[2].(error)
+	return ret0, ret1, ret2
+}
+
+// AcquireSector indicates an expected call of AcquireSector.
+func (mr *MockStoreMockRecorder) AcquireSector(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireSector", reflect.TypeOf((*MockStore)(nil).AcquireSector), arg0, arg1, arg2, arg3, arg4, arg5)
+}
+
+// FsStat mocks base method.
+func (m *MockStore) FsStat(arg0 context.Context, arg1 stores.ID) (fsutil.FsStat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FsStat", arg0, arg1)
+	ret0, _ := ret[0].(fsutil.FsStat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FsStat indicates an expected call of FsStat.
+func (mr *MockStoreMockRecorder) FsStat(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FsStat", reflect.TypeOf((*MockStore)(nil).FsStat), arg0, arg1)
+}
+
+// MoveStorage mocks base method.
+func (m *MockStore) MoveStorage(arg0 context.Context, arg1 storage.SectorRef, arg2 storiface.SectorFileType) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MoveStorage", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// MoveStorage indicates an expected call of MoveStorage.
+func (mr *MockStoreMockRecorder) MoveStorage(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MoveStorage", reflect.TypeOf((*MockStore)(nil).MoveStorage), arg0, arg1, arg2)
+}
+
+// Remove mocks base method.
+func (m *MockStore) Remove(arg0 context.Context, arg1 abi.SectorID, arg2 storiface.SectorFileType, arg3 bool, arg4 []stores.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Remove", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Remove indicates an expected call of Remove.
+func (mr *MockStoreMockRecorder) Remove(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockStore)(nil).Remove), arg0, arg1, arg2, arg3, arg4)
+}
+
+// RemoveCopies mocks base method.
+func (m *MockStore) RemoveCopies(arg0 context.Context, arg1 abi.SectorID, arg2 storiface.SectorFileType) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RemoveCopies", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RemoveCopies indicates an expected call of RemoveCopies.
+func (mr *MockStoreMockRecorder) RemoveCopies(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveCopies", reflect.TypeOf((*MockStore)(nil).RemoveCopies), arg0, arg1, arg2)
+}
+
+// Reserve mocks base method.
+func (m *MockStore) Reserve(arg0 context.Context, arg1 storage.SectorRef, arg2 storiface.SectorFileType, arg3 storiface.SectorPaths, arg4 map[storiface.SectorFileType]int) (func(), error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Reserve", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(func())
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Reserve indicates an expected call of Reserve.
+func (mr *MockStoreMockRecorder) Reserve(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reserve", reflect.TypeOf((*MockStore)(nil).Reserve), arg0, arg1, arg2, arg3, arg4)
+}
@@ -1,212 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: interface.go
-
-// Package mocks is a generated GoMock package.
-package mocks
-
-import (
-	context "context"
-	os "os"
-	reflect "reflect"
-
-	abi "github.com/filecoin-project/go-state-types/abi"
-	fsutil "github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	partialfile "github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
-	stores "github.com/filecoin-project/lotus/extern/sector-storage/stores"
-	storiface "github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	storage "github.com/filecoin-project/specs-storage/storage"
-	gomock "github.com/golang/mock/gomock"
-)
-
-// MockpartialFileHandler is a mock of partialFileHandler interface.
-type MockpartialFileHandler struct {
-	ctrl     *gomock.Controller
-	recorder *MockpartialFileHandlerMockRecorder
-}
-
-// MockpartialFileHandlerMockRecorder is the mock recorder for MockpartialFileHandler.
-type MockpartialFileHandlerMockRecorder struct {
-	mock *MockpartialFileHandler
-}
-
-// NewMockpartialFileHandler creates a new mock instance.
-func NewMockpartialFileHandler(ctrl *gomock.Controller) *MockpartialFileHandler {
-	mock := &MockpartialFileHandler{ctrl: ctrl}
-	mock.recorder = &MockpartialFileHandlerMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockpartialFileHandler) EXPECT() *MockpartialFileHandlerMockRecorder {
-	return m.recorder
-}
-
-// Close mocks base method.
-func (m *MockpartialFileHandler) Close(pf *partialfile.PartialFile) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Close", pf)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Close indicates an expected call of Close.
-func (mr *MockpartialFileHandlerMockRecorder) Close(pf interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockpartialFileHandler)(nil).Close), pf)
-}
-
-// HasAllocated mocks base method.
-func (m *MockpartialFileHandler) HasAllocated(pf *partialfile.PartialFile, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "HasAllocated", pf, offset, size)
-	ret0, _ := ret[0].(bool)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// HasAllocated indicates an expected call of HasAllocated.
-func (mr *MockpartialFileHandlerMockRecorder) HasAllocated(pf, offset, size interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasAllocated", reflect.TypeOf((*MockpartialFileHandler)(nil).HasAllocated), pf, offset, size)
-}
-
-// OpenPartialFile mocks base method.
-func (m *MockpartialFileHandler) OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialfile.PartialFile, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "OpenPartialFile", maxPieceSize, path)
-	ret0, _ := ret[0].(*partialfile.PartialFile)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// OpenPartialFile indicates an expected call of OpenPartialFile.
-func (mr *MockpartialFileHandlerMockRecorder) OpenPartialFile(maxPieceSize, path interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenPartialFile", reflect.TypeOf((*MockpartialFileHandler)(nil).OpenPartialFile), maxPieceSize, path)
-}
-
-// Reader mocks base method.
-func (m *MockpartialFileHandler) Reader(pf *partialfile.PartialFile, offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (*os.File, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Reader", pf, offset, size)
-	ret0, _ := ret[0].(*os.File)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// Reader indicates an expected call of Reader.
-func (mr *MockpartialFileHandlerMockRecorder) Reader(pf, offset, size interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reader", reflect.TypeOf((*MockpartialFileHandler)(nil).Reader), pf, offset, size)
-}
-
-// MockStore is a mock of Store interface.
-type MockStore struct {
-	ctrl     *gomock.Controller
-	recorder *MockStoreMockRecorder
-}
-
-// MockStoreMockRecorder is the mock recorder for MockStore.
-type MockStoreMockRecorder struct {
-	mock *MockStore
-}
-
-// NewMockStore creates a new mock instance.
-func NewMockStore(ctrl *gomock.Controller) *MockStore {
-	mock := &MockStore{ctrl: ctrl}
-	mock.recorder = &MockStoreMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockStore) EXPECT() *MockStoreMockRecorder {
-	return m.recorder
-}
-
-// AcquireSector mocks base method.
-func (m *MockStore) AcquireSector(ctx context.Context, s storage.SectorRef, existing, allocate storiface.SectorFileType, sealing storiface.PathType, op storiface.AcquireMode) (storiface.SectorPaths, storiface.SectorPaths, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "AcquireSector", ctx, s, existing, allocate, sealing, op)
-	ret0, _ := ret[0].(storiface.SectorPaths)
-	ret1, _ := ret[1].(storiface.SectorPaths)
-	ret2, _ := ret[2].(error)
-	return ret0, ret1, ret2
-}
-
-// AcquireSector indicates an expected call of AcquireSector.
-func (mr *MockStoreMockRecorder) AcquireSector(ctx, s, existing, allocate, sealing, op interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireSector", reflect.TypeOf((*MockStore)(nil).AcquireSector), ctx, s, existing, allocate, sealing, op)
-}
-
-// FsStat mocks base method.
-func (m *MockStore) FsStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "FsStat", ctx, id)
-	ret0, _ := ret[0].(fsutil.FsStat)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// FsStat indicates an expected call of FsStat.
-func (mr *MockStoreMockRecorder) FsStat(ctx, id interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FsStat", reflect.TypeOf((*MockStore)(nil).FsStat), ctx, id)
-}
-
-// MoveStorage mocks base method.
-func (m *MockStore) MoveStorage(ctx context.Context, s storage.SectorRef, types storiface.SectorFileType) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "MoveStorage", ctx, s, types)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// MoveStorage indicates an expected call of MoveStorage.
-func (mr *MockStoreMockRecorder) MoveStorage(ctx, s, types interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MoveStorage", reflect.TypeOf((*MockStore)(nil).MoveStorage), ctx, s, types)
-}
-
-// Remove mocks base method.
-func (m *MockStore) Remove(ctx context.Context, s abi.SectorID, types storiface.SectorFileType, force bool) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Remove", ctx, s, types, force)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Remove indicates an expected call of Remove.
-func (mr *MockStoreMockRecorder) Remove(ctx, s, types, force interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockStore)(nil).Remove), ctx, s, types, force)
-}
-
-// RemoveCopies mocks base method.
-func (m *MockStore) RemoveCopies(ctx context.Context, s abi.SectorID, types storiface.SectorFileType) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "RemoveCopies", ctx, s, types)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// RemoveCopies indicates an expected call of RemoveCopies.
-func (mr *MockStoreMockRecorder) RemoveCopies(ctx, s, types interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveCopies", reflect.TypeOf((*MockStore)(nil).RemoveCopies), ctx, s, types)
-}
-
-// Reserve mocks base method.
-func (m *MockStore) Reserve(ctx context.Context, sid storage.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Reserve", ctx, sid, ft, storageIDs, overheadTab)
-	ret0, _ := ret[0].(func())
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// Reserve indicates an expected call of Reserve.
-func (mr *MockStoreMockRecorder) Reserve(ctx, sid, ft, storageIDs, overheadTab interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reserve", reflect.TypeOf((*MockStore)(nil).Reserve), ctx, sid, ft, storageIDs, overheadTab)
-}
@@ -41,7 +41,7 @@ type Remote struct {
 	fetchLk  sync.Mutex
 	fetching map[abi.SectorID]chan struct{}

-	pfHandler partialFileHandler
+	pfHandler PartialFileHandler
 }

 func (r *Remote) RemoveCopies(ctx context.Context, s abi.SectorID, types storiface.SectorFileType) error {
@@ -52,7 +52,7 @@ func (r *Remote) RemoveCopies(ctx context.Context, s abi.SectorID, types storifa
 	return r.local.RemoveCopies(ctx, s, types)
 }

-func NewRemote(local Store, index SectorIndex, auth http.Header, fetchLimit int, pfHandler partialFileHandler) *Remote {
+func NewRemote(local Store, index SectorIndex, auth http.Header, fetchLimit int, pfHandler PartialFileHandler) *Remote {
 	return &Remote{
 		local: local,
 		index: index,
@@ -155,7 +155,8 @@ func (r *Remote) AcquireSector(ctx context.Context, s storage.SectorRef, existin
 		}

 		if op == storiface.AcquireMove {
-			if err := r.deleteFromRemote(ctx, url); err != nil {
+			id := ID(storageID)
+			if err := r.deleteFromRemote(ctx, url, &id); err != nil {
 				log.Warnf("deleting sector %v from %s (delete %s): %+v", s, storageID, url, err)
 			}
 		}
@@ -280,7 +281,7 @@ func (r *Remote) fetch(ctx context.Context, url, outname string) error {

 	switch mediatype {
 	case "application/x-tar":
-		return tarutil.ExtractTar(resp.Body, outname)
+		return tarutil.ExtractTar(resp.Body, outname, make([]byte, CopyBuf))
 	case "application/octet-stream":
 		f, err := os.Create(outname)
 		if err != nil {
@@ -304,7 +305,6 @@ func (r *Remote) checkAllocated(ctx context.Context, url string, spt abi.Registe
 		return false, xerrors.Errorf("request: %w", err)
 	}
 	req.Header = r.auth.Clone()
-	fmt.Printf("req using header: %#v \n", r.auth)
 	req = req.WithContext(ctx)

 	resp, err := http.DefaultClient.Do(req)
@@ -333,12 +333,12 @@ func (r *Remote) MoveStorage(ctx context.Context, s storage.SectorRef, types sto
 	return r.local.MoveStorage(ctx, s, types)
 }

-func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool) error {
+func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool, keepIn []ID) error {
 	if bits.OnesCount(uint(typ)) != 1 {
 		return xerrors.New("delete expects one file type")
 	}

-	if err := r.local.Remove(ctx, sid, typ, force); err != nil {
+	if err := r.local.Remove(ctx, sid, typ, force, keepIn); err != nil {
 		return xerrors.Errorf("remove from local: %w", err)
 	}

@@ -347,9 +347,15 @@ func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.Sec
 		return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err)
 	}

+storeLoop:
 	for _, info := range si {
+		for _, id := range keepIn {
+			if id == info.ID {
+				continue storeLoop
+			}
+		}
 		for _, url := range info.URLs {
-			if err := r.deleteFromRemote(ctx, url); err != nil {
+			if err := r.deleteFromRemote(ctx, url, nil); err != nil {
 				log.Warnf("remove %s: %+v", url, err)
 				continue
 			}
@@ -360,7 +366,11 @@ func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.Sec
 	return nil
 }

-func (r *Remote) deleteFromRemote(ctx context.Context, url string) error {
+func (r *Remote) deleteFromRemote(ctx context.Context, url string, keepIn *ID) error {
+	if keepIn != nil {
+		url = url + "?keep=" + string(*keepIn)
+	}
+
 	log.Infof("Delete %s", url)

 	req, err := http.NewRequest("DELETE", url, nil)
@@ -575,7 +585,7 @@ func (r *Remote) CheckIsUnsealed(ctx context.Context, s storage.SectorRef, offse
 // 1. no worker(local worker included) has an unsealed file for the given sector OR
 // 2. no worker(local worker included) has the unsealed piece in their unsealed sector file.
 // Will return a nil reader and a nil error in such a case.
-func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize) (io.ReadCloser, error) {
+func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize) (func(startOffsetAligned storiface.PaddedByteIndex) (io.ReadCloser, error), error) {
 	ft := storiface.FTUnsealed

 	// check if we have the unsealed sector file locally
@@ -613,7 +623,52 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a

 		if has {
 			log.Infof("returning piece reader for local unsealed piece sector=%+v, (offset=%d, size=%d)", s.ID, offset, size)
-			return r.pfHandler.Reader(pf, storiface.PaddedByteIndex(offset), size)
+
+			return func(startOffsetAligned storiface.PaddedByteIndex) (io.ReadCloser, error) {
+				// don't reuse between readers unless closed
+				f := pf
+				pf = nil
+
+				if f == nil {
+					f, err = r.pfHandler.OpenPartialFile(abi.PaddedPieceSize(ssize), path)
+					if err != nil {
+						return nil, xerrors.Errorf("opening partial file: %w", err)
+					}
+					log.Debugf("local partial file (re)opened %s (+%d,%d)", path, offset, size)
+				}
+
+				r, err := r.pfHandler.Reader(f, storiface.PaddedByteIndex(offset)+startOffsetAligned, size-abi.PaddedPieceSize(startOffsetAligned))
+				if err != nil {
+					return nil, err
+				}
+
+				return struct {
+					io.Reader
+					io.Closer
+				}{
+					Reader: r,
+					Closer: funcCloser(func() error {
+						// if we already have a reader cached, close this one
+						if pf != nil {
+							if f == nil {
+								return nil
+							}
+							if pf == f {
+								pf = nil
+							}
+
+							tmp := f
+							f = nil
+							return tmp.Close()
+						}
+
+						// otherwise stash it away for reuse
+						pf = f
+						return nil
+					}),
+				}, nil
+			}, nil
+
 		}

 		log.Debugf("miner has unsealed file but not unseal piece, %s (+%d,%d)", path, offset, size)
@@ -656,16 +711,18 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a
 				continue
 			}

-			// readRemote fetches a reader that we can use to read the unsealed piece from the remote worker.
-			// It uses a ranged HTTP query to ensure we ONLY read the unsealed piece and not the entire unsealed file.
-			rd, err := r.readRemote(ctx, url, offset, size)
-			if err != nil {
-				log.Warnw("reading from remote", "url", url, "error", err)
-				lastErr = err
-				continue
-			}
-			log.Infof("Read remote %s (+%d,%d)", url, offset, size)
-			return rd, nil
+			return func(startOffsetAligned storiface.PaddedByteIndex) (io.ReadCloser, error) {
+				// readRemote fetches a reader that we can use to read the unsealed piece from the remote worker.
+				// It uses a ranged HTTP query to ensure we ONLY read the unsealed piece and not the entire unsealed file.
+				rd, err := r.readRemote(ctx, url, offset+abi.PaddedPieceSize(startOffsetAligned), size)
+				if err != nil {
+					log.Warnw("reading from remote", "url", url, "error", err)
+					return nil, err
+				}
+
+				return rd, err
+			}, nil
+
 		}
 	}

@@ -682,3 +739,11 @@ func (r *Remote) Reserve(ctx context.Context, sid storage.SectorRef, ft storifac
 }

 var _ Store = &Remote{}
+
+type funcCloser func() error
+
+func (f funcCloser) Close() error {
+	return f()
+}
+
+var _ io.Closer = funcCloser(nil)
@@ -2,26 +2,157 @@ package stores_test

 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"testing"

+	"github.com/golang/mock/gomock"
+	"github.com/google/uuid"
+	"github.com/gorilla/mux"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores/mocks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/lotus/node/repo"
 	"github.com/filecoin-project/specs-storage/storage"
-	"github.com/golang/mock/gomock"
-	"github.com/gorilla/mux"
-	logging "github.com/ipfs/go-log/v2"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
 )

+const metaFile = "sectorstore.json"
+
+func createTestStorage(t *testing.T, p string, seal bool, att ...*stores.Local) stores.ID {
+	if err := os.MkdirAll(p, 0755); err != nil {
+		if !os.IsExist(err) {
+			require.NoError(t, err)
+		}
+	}
+
+	cfg := &stores.LocalStorageMeta{
+		ID:       stores.ID(uuid.New().String()),
+		Weight:   10,
+		CanSeal:  seal,
+		CanStore: !seal,
+	}
+
+	b, err := json.MarshalIndent(cfg, "", "  ")
+	require.NoError(t, err)
+
+	require.NoError(t, ioutil.WriteFile(filepath.Join(p, metaFile), b, 0644))
+
+	for _, s := range att {
+		require.NoError(t, s.OpenPath(context.Background(), p))
+	}
+
+	return cfg.ID
+}
+
+func TestMoveShared(t *testing.T) {
+	logging.SetAllLoggers(logging.LevelDebug)
+
+	index := stores.NewIndex()
+
+	ctx := context.Background()
+
+	dir, err := ioutil.TempDir("", "stores-remote-test-")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll(dir)
+	})
+
+	openRepo := func(dir string) repo.LockedRepo {
+		r, err := repo.NewFS(dir)
+		require.NoError(t, err)
+		require.NoError(t, r.Init(repo.Worker))
+		lr, err := r.Lock(repo.Worker)
+		require.NoError(t, err)
+
+		t.Cleanup(func() {
+			_ = lr.Close()
+		})
+
+		err = lr.SetStorage(func(config *stores.StorageConfig) {
+			*config = stores.StorageConfig{}
+		})
+		require.NoError(t, err)
+
+		return lr
+	}
+
+	// setup two repos with two storage paths:
+	// repo 1 with both paths
+	// repo 2 with one path (shared)
+
+	lr1 := openRepo(filepath.Join(dir, "l1"))
+	lr2 := openRepo(filepath.Join(dir, "l2"))
+
+	mux1 := mux.NewRouter()
+	mux2 := mux.NewRouter()
+	hs1 := httptest.NewServer(mux1)
+	hs2 := httptest.NewServer(mux2)
+
+	ls1, err := stores.NewLocal(ctx, lr1, index, []string{hs1.URL + "/remote"})
+	require.NoError(t, err)
+	ls2, err := stores.NewLocal(ctx, lr2, index, []string{hs2.URL + "/remote"})
+	require.NoError(t, err)
+
+	dirStor := filepath.Join(dir, "stor")
+	dirSeal := filepath.Join(dir, "seal")
+
+	id1 := createTestStorage(t, dirStor, false, ls1, ls2)
+	id2 := createTestStorage(t, dirSeal, true, ls1)
+
+	rs1 := stores.NewRemote(ls1, index, nil, 20, &stores.DefaultPartialFileHandler{})
+	rs2 := stores.NewRemote(ls2, index, nil, 20, &stores.DefaultPartialFileHandler{})
+	_ = rs2
+	mux1.PathPrefix("/").Handler(&stores.FetchHandler{Local: ls1, PfHandler: &stores.DefaultPartialFileHandler{}})
+	mux2.PathPrefix("/").Handler(&stores.FetchHandler{Local: ls2, PfHandler: &stores.DefaultPartialFileHandler{}})
+
+	// add a sealed replica file to the sealing (non-shared) path
+
+	s1ref := storage.SectorRef{
+		ID: abi.SectorID{
+			Miner:  12,
+			Number: 1,
+		},
+		ProofType: abi.RegisteredSealProof_StackedDrg2KiBV1,
+	}
+
+	sp, sid, err := rs1.AcquireSector(ctx, s1ref, storiface.FTNone, storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove)
+	require.NoError(t, err)
+	require.Equal(t, id2, stores.ID(sid.Sealed))
+
+	data := make([]byte, 2032)
+	data[1] = 54
+	require.NoError(t, ioutil.WriteFile(sp.Sealed, data, 0666))
+	fmt.Println("write to ", sp.Sealed)
+
+	require.NoError(t, index.StorageDeclareSector(ctx, stores.ID(sid.Sealed), s1ref.ID, storiface.FTSealed, true))
+
+	// move to the shared path from the second node (remote move / delete)
+
+	require.NoError(t, rs2.MoveStorage(ctx, s1ref, storiface.FTSealed))
+
+	// check that the file still exists
+	sp, sid, err = rs2.AcquireSector(ctx, s1ref, storiface.FTSealed, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
+	require.NoError(t, err)
+	require.Equal(t, id1, stores.ID(sid.Sealed))
+	fmt.Println("read from ", sp.Sealed)
+
+	read, err := ioutil.ReadFile(sp.Sealed)
+	require.NoError(t, err)
+	require.EqualValues(t, data, read)
+}
+
 func TestReader(t *testing.T) {
 	logging.SetAllLoggers(logging.LevelDebug)
 	bz := []byte("Hello World")
@@ -46,7 +177,7 @@ func TestReader(t *testing.T) {

 	tcs := map[string]struct {
 		storeFnc func(s *mocks.MockStore)
-		pfFunc   func(s *mocks.MockpartialFileHandler)
+		pfFunc   func(s *mocks.MockPartialFileHandler)
 		indexFnc func(s *mocks.MockSectorIndex, serverURL string)

 		needHttpServer bool
@@ -76,7 +207,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, xerrors.New("pf open error"))
 			},
 			errStr: "pf open error",
@@ -87,7 +218,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, xerrors.New("piece check error"))
@@ -101,7 +232,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -115,7 +246,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, nil)
@@ -157,7 +288,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -223,7 +354,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, nil)
@@ -251,7 +382,7 @@ func TestReader(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -308,7 +439,7 @@ func TestReader(t *testing.T) {

 			// create them mocks
 			lstore := mocks.NewMockStore(mockCtrl)
-			pfhandler := mocks.NewMockpartialFileHandler(mockCtrl)
+			pfhandler := mocks.NewMockPartialFileHandler(mockCtrl)
 			index := mocks.NewMockSectorIndex(mockCtrl)

 			if tc.storeFnc != nil {
@@ -340,12 +471,20 @@ func TestReader(t *testing.T) {

 			remoteStore := stores.NewRemote(lstore, index, nil, 6000, pfhandler)

-			rd, err := remoteStore.Reader(ctx, sectorRef, offset, size)
+			rdg, err := remoteStore.Reader(ctx, sectorRef, offset, size)
+			var rd io.ReadCloser

 			if tc.errStr != "" {
-				require.Error(t, err)
-				require.Nil(t, rd)
-				require.Contains(t, err.Error(), tc.errStr)
+				if rdg == nil {
+					require.Error(t, err)
+					require.Nil(t, rdg)
+					require.Contains(t, err.Error(), tc.errStr)
+				} else {
+					rd, err = rdg(0)
+					require.Error(t, err)
+					require.Nil(t, rd)
+					require.Contains(t, err.Error(), tc.errStr)
+				}
 			} else {
 				require.NoError(t, err)
 			}
@@ -353,7 +492,10 @@ func TestReader(t *testing.T) {
 			if !tc.expectedNonNilReader {
 				require.Nil(t, rd)
 			} else {
-				require.NotNil(t, rd)
+				require.NotNil(t, rdg)
+				rd, err := rdg(0)
+				require.NoError(t, err)
+
 				defer func() {
 					require.NoError(t, rd.Close())
 				}()
@@ -393,7 +535,7 @@ func TestCheckIsUnsealed(t *testing.T) {

 	tcs := map[string]struct {
 		storeFnc func(s *mocks.MockStore)
-		pfFunc   func(s *mocks.MockpartialFileHandler)
+		pfFunc   func(s *mocks.MockPartialFileHandler)
 		indexFnc func(s *mocks.MockSectorIndex, serverURL string)

 		needHttpServer bool
@@ -421,7 +563,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, xerrors.New("pf open error"))
 			},
 			errStr: "pf open error",
@@ -432,7 +574,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, xerrors.New("piece check error"))
@@ -446,7 +588,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)

 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
@@ -488,7 +630,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -533,7 +675,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					true, nil)
@@ -569,7 +711,7 @@ func TestCheckIsUnsealed(t *testing.T) {
 				mockSectorAcquire(l, sectorRef, pfPath, nil)
 			},

-			pfFunc: func(pf *mocks.MockpartialFileHandler) {
+			pfFunc: func(pf *mocks.MockPartialFileHandler) {
 				mockPartialFileOpen(pf, sectorSize, pfPath, nil)
 				mockCheckAllocation(pf, offset, size, emptyPartialFile,
 					false, nil)
@@ -602,7 +744,7 @@ func TestCheckIsUnsealed(t *testing.T) {

 			// create them mocks
 			lstore := mocks.NewMockStore(mockCtrl)
-			pfhandler := mocks.NewMockpartialFileHandler(mockCtrl)
+			pfhandler := mocks.NewMockPartialFileHandler(mockCtrl)
 			index := mocks.NewMockSectorIndex(mockCtrl)

 			if tc.storeFnc != nil {
@@ -656,18 +798,18 @@ func mockSectorAcquire(l *mocks.MockStore, sectorRef storage.SectorRef, pfPath s
 		storiface.SectorPaths{}, err).Times(1)
 }

-func mockPartialFileOpen(pf *mocks.MockpartialFileHandler, sectorSize abi.SectorSize, pfPath string, err error) {
+func mockPartialFileOpen(pf *mocks.MockPartialFileHandler, sectorSize abi.SectorSize, pfPath string, err error) {
 	pf.EXPECT().OpenPartialFile(abi.PaddedPieceSize(sectorSize), pfPath).Return(&partialfile.PartialFile{},
 		err).Times(1)
 }

-func mockCheckAllocation(pf *mocks.MockpartialFileHandler, offset, size abi.PaddedPieceSize, file *partialfile.PartialFile,
+func mockCheckAllocation(pf *mocks.MockPartialFileHandler, offset, size abi.PaddedPieceSize, file *partialfile.PartialFile,
 	out bool, err error) {
 	pf.EXPECT().HasAllocated(file, storiface.UnpaddedByteIndex(offset.Unpadded()),
 		size.Unpadded()).Return(out, err).Times(1)
 }

-func mockPfReader(pf *mocks.MockpartialFileHandler, file *partialfile.PartialFile, offset, size abi.PaddedPieceSize,
+func mockPfReader(pf *mocks.MockPartialFileHandler, file *partialfile.PartialFile, offset, size abi.PaddedPieceSize,
 	outFile *os.File, err error) {
 	pf.EXPECT().Reader(file, storiface.PaddedByteIndex(offset), size).Return(outFile, err)
 }
@@ -1,19 +1,31 @@
-package sectorstorage
+package storiface

 import (
-	"github.com/filecoin-project/go-state-types/abi"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"

+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 )

 type Resources struct {
-	MinMemory uint64 // What Must be in RAM for decent perf
-	MaxMemory uint64 // Memory required (swap + ram)
+	MinMemory uint64 `envname:"MIN_MEMORY"` // What Must be in RAM for decent perf
+	MaxMemory uint64 `envname:"MAX_MEMORY"` // Memory required (swap + ram; peak memory usage during task execution)

-	MaxParallelism int // -1 = multithread
-	CanGPU         bool
+	// GPUUtilization specifes the number of GPUs a task can use
+	GPUUtilization float64 `envname:"GPU_UTILIZATION"`

-	BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads)
+	// MaxParallelism specifies the number of CPU cores when GPU is NOT in use
+	MaxParallelism int `envname:"MAX_PARALLELISM"` // -1 = multithread
+
+	// MaxParallelismGPU specifies the number of CPU cores when GPU is in use
+	MaxParallelismGPU int `envname:"MAX_PARALLELISM_GPU"` // when 0, inherits MaxParallelism
+
+	BaseMinMemory uint64 `envname:"BASE_MIN_MEMORY"` // What Must be in RAM for decent perf (shared between threads)
 }

 /*
@@ -32,8 +44,14 @@ var ParallelNum uint64 = 92
 var ParallelDenom uint64 = 100

 // TODO: Take NUMA into account
-func (r Resources) Threads(wcpus uint64) uint64 {
-	if r.MaxParallelism == -1 {
+func (r Resources) Threads(wcpus uint64, gpus int) uint64 {
+	mp := r.MaxParallelism
+
+	if r.GPUUtilization > 0 && gpus > 0 && r.MaxParallelismGPU != 0 { // task can use GPUs and worker has some
+		mp = r.MaxParallelismGPU
+	}
+
+	if mp == -1 {
 		n := (wcpus * ParallelNum) / ParallelDenom
 		if n == 0 {
 			return wcpus
@@ -41,7 +59,7 @@ func (r Resources) Threads(wcpus uint64) uint64 {
 		return n
 	}

-	return uint64(r.MaxParallelism)
+	return uint64(mp)
 }

 var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{
@@ -134,8 +152,9 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MaxMemory: 30 << 30,
 			MinMemory: 30 << 30,

-			MaxParallelism: -1,
-			CanGPU:         true,
+			MaxParallelism:    -1,
+			MaxParallelismGPU: 6,
+			GPUUtilization:    1.0,

 			BaseMinMemory: 1 << 30,
 		},
@@ -143,8 +162,9 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MaxMemory: 15 << 30,
 			MinMemory: 15 << 30,

-			MaxParallelism: -1,
-			CanGPU:         true,
+			MaxParallelism:    -1,
+			MaxParallelismGPU: 6,
+			GPUUtilization:    1.0,

 			BaseMinMemory: 1 << 30,
 		},
@@ -220,8 +240,9 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MaxMemory: 190 << 30, // TODO: Confirm
 			MinMemory: 60 << 30,

-			MaxParallelism: -1,
-			CanGPU:         true,
+			MaxParallelism:    -1,
+			MaxParallelismGPU: 6,
+			GPUUtilization:    1.0,

 			BaseMinMemory: 64 << 30, // params
 		},
@@ -229,8 +250,9 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory
 			MinMemory: 30 << 30,

-			MaxParallelism: -1,
-			CanGPU:         true,
+			MaxParallelism:    -1,
+			MaxParallelismGPU: 6,
+			GPUUtilization:    1.0,

 			BaseMinMemory: 32 << 30, // params
 		},
@@ -239,7 +261,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MinMemory: 1 << 30,

 			MaxParallelism: 1, // This is fine
-			CanGPU:         true,
+			GPUUtilization: 1.0,

 			BaseMinMemory: 10 << 30,
 		},
@@ -248,7 +270,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MinMemory: 2 << 10,

 			MaxParallelism: 1,
-			CanGPU:         true,
+			GPUUtilization: 1.0,

 			BaseMinMemory: 2 << 10,
 		},
@@ -257,7 +279,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MinMemory: 8 << 20,

 			MaxParallelism: 1,
-			CanGPU:         true,
+			GPUUtilization: 1.0,

 			BaseMinMemory: 8 << 20,
 		},
@@ -268,7 +290,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MinMemory: 1 << 20,

 			MaxParallelism: 0,
-			CanGPU:         false,
+			GPUUtilization: 0,

 			BaseMinMemory: 0,
 		},
@@ -277,7 +299,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MinMemory: 1 << 20,

 			MaxParallelism: 0,
-			CanGPU:         false,
+			GPUUtilization: 0,

 			BaseMinMemory: 0,
 		},
@@ -286,7 +308,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MinMemory: 1 << 20,

 			MaxParallelism: 0,
-			CanGPU:         false,
+			GPUUtilization: 0,

 			BaseMinMemory: 0,
 		},
@@ -295,7 +317,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MinMemory: 1 << 20,

 			MaxParallelism: 0,
-			CanGPU:         false,
+			GPUUtilization: 0,

 			BaseMinMemory: 0,
 		},
@@ -304,7 +326,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 			MinMemory: 1 << 20,

 			MaxParallelism: 0,
-			CanGPU:         false,
+			GPUUtilization: 0,

 			BaseMinMemory: 0,
 		},
@@ -323,3 +345,83 @@ func init() {
 		m[abi.RegisteredSealProof_StackedDrg64GiBV1_1] = m[abi.RegisteredSealProof_StackedDrg64GiBV1]
 	}
 }
+
+func ParseResourceEnv(lookup func(key, def string) (string, bool)) (map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources, error) {
+	out := map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{}
+
+	for taskType, defTT := range ResourceTable {
+		out[taskType] = map[abi.RegisteredSealProof]Resources{}
+
+		for spt, defRes := range defTT {
+			r := defRes // copy
+
+			spsz, err := spt.SectorSize()
+			if err != nil {
+				return nil, xerrors.Errorf("getting sector size: %w", err)
+			}
+			shortSize := strings.TrimSuffix(spsz.ShortString(), "iB")
+
+			rr := reflect.ValueOf(&r)
+			for i := 0; i < rr.Elem().Type().NumField(); i++ {
+				f := rr.Elem().Type().Field(i)
+
+				envname := f.Tag.Get("envname")
+				if envname == "" {
+					return nil, xerrors.Errorf("no envname for field '%s'", f.Name)
+				}
+
+				envval, found := lookup(taskType.Short()+"_"+shortSize+"_"+envname, fmt.Sprint(rr.Elem().Field(i).Interface()))
+				if !found {
+					// special multicore SDR handling
+					if (taskType == sealtasks.TTPreCommit1 || taskType == sealtasks.TTUnseal) && envname == "MAX_PARALLELISM" {
+						v, ok := rr.Elem().Field(i).Addr().Interface().(*int)
+						if !ok {
+							// can't happen, but let's not panic
+							return nil, xerrors.Errorf("res.MAX_PARALLELISM is not int (!?): %w", err)
+						}
+						*v, err = getSDRThreads(lookup)
+						if err != nil {
+							return nil, err
+						}
+					}
+
+					continue
+				}
+
+				v := rr.Elem().Field(i).Addr().Interface()
+				switch fv := v.(type) {
+				case *uint64:
+					*fv, err = strconv.ParseUint(envval, 10, 64)
+				case *int:
+					*fv, err = strconv.Atoi(envval)
+				case *float64:
+					*fv, err = strconv.ParseFloat(envval, 64)
+				default:
+					return nil, xerrors.Errorf("unknown resource field type")
+				}
+			}
+
+			out[taskType][spt] = r
+		}
+	}
+
+	return out, nil
+}
+
+func getSDRThreads(lookup func(key, def string) (string, bool)) (_ int, err error) {
+	producers := 0
+
+	if v, _ := lookup("FIL_PROOFS_USE_MULTICORE_SDR", ""); v == "1" {
+		producers = 3
+
+		if penv, found := lookup("FIL_PROOFS_MULTICORE_SDR_PRODUCERS", ""); found {
+			producers, err = strconv.Atoi(penv)
+			if err != nil {
+				return 0, xerrors.Errorf("parsing (atoi) FIL_PROOFS_MULTICORE_SDR_PRODUCERS: %w", err)
+			}
+		}
+	}
+
+	// producers + the one core actually doing the work
+	return producers + 1, nil
+}
@@ -0,0 +1,75 @@
+package storiface
+
+import (
+	"fmt"
+	"testing"
+
+	stabi "github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListResourceVars(t *testing.T) {
+	_, err := ParseResourceEnv(func(key, def string) (string, bool) {
+		if def != "" {
+			fmt.Printf("%s=%s\n", key, def)
+		}
+
+		return "", false
+	})
+
+	require.NoError(t, err)
+}
+
+func TestListResourceOverride(t *testing.T) {
+	rt, err := ParseResourceEnv(func(key, def string) (string, bool) {
+		if key == "UNS_2K_MAX_PARALLELISM" {
+			return "2", true
+		}
+		if key == "PC2_2K_GPU_UTILIZATION" {
+			return "0.4", true
+		}
+		if key == "PC2_2K_MAX_MEMORY" {
+			return "2222", true
+		}
+
+		return "", false
+	})
+
+	require.NoError(t, err)
+	require.Equal(t, 2, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+	require.Equal(t, 0.4, rt[sealtasks.TTPreCommit2][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].GPUUtilization)
+	require.Equal(t, uint64(2222), rt[sealtasks.TTPreCommit2][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxMemory)
+
+	// check that defaults don't get mutated
+	require.Equal(t, 1, ResourceTable[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+}
+
+func TestListResourceSDRMulticoreOverride(t *testing.T) {
+	rt, err := ParseResourceEnv(func(key, def string) (string, bool) {
+		if key == "FIL_PROOFS_USE_MULTICORE_SDR" {
+			return "1", true
+		}
+
+		return "", false
+	})
+
+	require.NoError(t, err)
+	require.Equal(t, 4, rt[sealtasks.TTPreCommit1][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+	require.Equal(t, 4, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+
+	rt, err = ParseResourceEnv(func(key, def string) (string, bool) {
+		if key == "FIL_PROOFS_USE_MULTICORE_SDR" {
+			return "1", true
+		}
+		if key == "FIL_PROOFS_MULTICORE_SDR_PRODUCERS" {
+			return "9000", true
+		}
+
+		return "", false
+	})
+
+	require.NoError(t, err)
+	require.Equal(t, 9001, rt[sealtasks.TTPreCommit1][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+	require.Equal(t, 9001, rt[sealtasks.TTUnseal][stabi.RegisteredSealProof_StackedDrg2KiBV1_1].MaxParallelism)
+}
@@ -15,6 +15,12 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 )

+type WorkerID uuid.UUID // worker session UUID
+
+func (w WorkerID) String() string {
+	return uuid.UUID(w).String()
+}
+
 type WorkerInfo struct {
 	Hostname string

@@ -28,12 +34,35 @@ type WorkerInfo struct {

 type WorkerResources struct {
 	MemPhysical uint64
+	MemUsed     uint64
 	MemSwap     uint64
-
-	MemReserved uint64 // Used by system / other processes
+	MemSwapUsed uint64

 	CPUs uint64 // Logical cores
 	GPUs []string
+
+	// if nil use the default resource table
+	Resources map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
+}
+
+func (wr WorkerResources) ResourceSpec(spt abi.RegisteredSealProof, tt sealtasks.TaskType) Resources {
+	res := ResourceTable[tt][spt]
+
+	// if the worker specifies custom resource table, prefer that
+	if wr.Resources != nil {
+		tr, ok := wr.Resources[tt]
+		if !ok {
+			return res
+		}
+
+		r, ok := tr[spt]
+		if ok {
+			return r
+		}
+	}
+
+	// otherwise, use the default resource table
+	return res
 }

 type WorkerStats struct {
@@ -42,11 +71,13 @@ type WorkerStats struct {

 	MemUsedMin uint64
 	MemUsedMax uint64
-	GpuUsed    bool   // nolint
-	CpuUse     uint64 // nolint
+	GpuUsed    float64 // nolint
+	CpuUse     uint64  // nolint
 }

 const (
+	RWPrepared = 1
+	RWRunning  = 0
 	RWRetWait  = -1
 	RWReturned = -2
 	RWRetDone  = -3
@@ -57,7 +88,8 @@ type WorkerJob struct {
 	Sector abi.SectorID
 	Task   sealtasks.TaskType

-	// 1+ - assigned
+	// 2+ - assigned
+	// 1  - prepared
 	// 0  - running
 	// -1 - ret-wait
 	// -2 - returned
--- a/Show More
+++ b/Show More