Merge pull request #5775 from filecoin-project/feat/release-v1.5.2

Lotus v1.5.2

.circleci/config.yml

@@ -2,6 +2,7 @@ version: 2.1
 orbs:
   go: gotest/tools@0.0.13
   aws-cli: circleci/aws-cli@1.3.2
+  packer: salaxander/packer@0.0.3
 
 executors:
   golang:
@@ -277,6 +278,11 @@ jobs:
       - install-deps
       - prepare
       - run: make calibnet
+      - run: mkdir linux-calibnet && mv lotus lotus-miner lotus-worker linux-calibnet
+      - persist_to_workspace:
+          root: "."
+          paths:
+            - linux-calibnet
   build-lotus-soup:
     description: |
       Compile `lotus-soup` Testground test plan
@@ -289,7 +295,7 @@ jobs:
       - run: cd extern/filecoin-ffi && make
       - run:
           name: "go get lotus@master"
-          command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
+          command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../..
       - run:
           name: "build lotus-soup testplan"
           command: pushd testplans/lotus-soup && go build -tags=testground .
@@ -583,6 +589,22 @@ jobs:
               docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
             done
 
+  publish-packer:
+    description: build and push AWS IAM and DigitalOcean droplet.
+    executor:
+      name: packer/default
+      packer-version: 1.6.6
+    steps:
+      - checkout
+      - attach_workspace:
+          at: "."
+      - packer/build:
+          template: tools/packer/lotus.pkr.hcl
+          args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG"
+      - packer/build:
+          template: tools/packer/lotus.pkr.hcl
+          args: "-var ci_workspace_bins=./linux-calibnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG"
+
 workflows:
   version: 2.1
   ci:
@@ -683,3 +705,15 @@ workflows:
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
+      - publish-packer:
+          requires:
+            - build-all
+            - build-ntwk-calibration
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+         

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,33 @@
+---
+name: Bug Report
+about: Create a report to help us improve
+title: "[BUG] "
+labels: hint/needs-triaging, kind/bug
+assignees: ''
+
+---
+
+> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
+
+**Describe the bug**
+A clear and concise description of what the bug is. 
+(If you are not sure what the bug is, try to figure it out via a [discussion](https://github.com/filecoin-project/lotus/discussions/new) first!
+
+**Version (run `lotus version`):**
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Run '...'
+2. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Logs**
+Provide daemon/miner/worker logs, and goroutines(if available) for troubleshooting.
+ 
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,27 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Run '...'
-2. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Version (run `lotus version`):**
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/deal-making-issues.md

@@ -0,0 +1,49 @@
+---
+name: Deal Making Issues
+about: Create a report for help with deal making failures.
+title: "[Deal Making Issue]"
+labels: hint/needs-triaging, area/markets
+assignees: ''
+
+---
+
+> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
+
+Please provide all the information requested here to help us troubleshoot "deal making failed" issues.
+If the information requested is missing, we will probably have to just ask you to provide it anyway,
+before we can help debug.
+
+**Basic Information**
+Including information like, Are you the client or the miner? Is this a storage deal or a retrieval deal? Is it an offline deal?
+
+**Describe the problem**
+
+A brief description of the problem you encountered while trying to make a deal. 
+
+**Version**
+
+The output of `lotus --version`.
+
+**Setup**
+
+You miner(if applicable) and daemon setup, i.e: What hardware do you use, how much ram and etc.
+
+**To Reproduce**
+ Steps to reproduce the behavior:
+ 1. Run '...'
+ 2. See error
+
+**Deal status**
+
+The output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
+
+**Lotus daemon and miner logs**
+
+Please go through the logs of your daemon and miner(if applicable), and include screenshots of any error/warning-like messages you find.
+
+Alternatively please upload full log files and share a link here
+
+** Code modifications **
+
+If you have modified parts of lotus, please describe which areas were modified,
+and the scope of those modifications

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[Feature Request]"
+labels: hint/needs-triaging
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/mining-issues.md

@@ -0,0 +1,35 @@
+---
+name: Mining Issues
+about: Create a report for help with mining failures.
+title: "[Mining Issue]"
+labels: hint/needs-triaging, area/mining
+assignees: ''
+
+---
+
+> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
+
+Please provide all the information requested here to help us troubleshoot "mining/WinningPoSt failed" issues.
+If the information requested is missing, you may be asked you to provide it.
+
+**Describe the problem**
+A brief description of the problem you encountered while mining new blocks.
+
+**Version**
+
+The output of `lotus --version`.
+
+**Setup**
+
+You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
+
+**Lotus daemon and miner logs**
+
+Please go through the logs of your daemon and miner, and include screenshots of any error/warning-like messages you find, highlighting the one has "winning post" in it.
+
+Alternatively please upload full log files and share a link here
+
+** Code modifications **
+
+If you have modified parts of lotus, please describe which areas were modified,
+and the scope of those modifications

.github/ISSUE_TEMPLATE/proving-issues.md

@@ -0,0 +1,46 @@
+---
+name: Proving Issues
+about: Create a report for help with proving failures.
+title: "[Proving Issue]"
+labels: area/proving, hint/needs-triaging
+assignees: ''
+
+---
+
+> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
+
+Please provide all the information requested here to help us troubleshoot "proving/window PoSt failed" issues.
+If the information requested is missing, we will probably have to just ask you to provide it anyway,
+before we can help debug.
+
+**Describe the problem**
+A brief description of the problem you encountered while proving the storage.
+
+**Version**
+
+The output of `lotus --version`.
+
+**Setup**
+
+You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
+
+**Proving status**
+
+The output of `lotus-miner proving` info.
+
+**Lotus miner logs**
+
+Please go through the logs of your miner, and include screenshots of any error-like messages you find, highlighting the one has "window post" in it.
+
+Alternatively please upload full log files and share a link here
+
+**Lotus miner diagnostic info**
+
+Please collect the following diagnostic information, and share a link here
+
+* lotus-miner diagnostic info `lotus-miner info all > allinfo.txt`
+
+** Code modifications **
+
+If you have modified parts of lotus, please describe which areas were modified,
+and the scope of those modifications

.github/ISSUE_TEMPLATE/sealing-issues.md

@@ -1,21 +1,32 @@
 ---
 name: Sealing Issues
 about: Create a report for help with sealing (commit) failures.
-title: ''
+title: "[Sealing Issue]"
-labels: 'sealing'
+labels: hint/needs-triaging, area/sealing
 assignees: ''
 
 ---
 
+> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
+
 Please provide all the information requested here to help us troubleshoot "commit failed" issues.
 If the information requested is missing, we will probably have to just ask you to provide it anyway,
 before we can help debug.
 
 **Describe the problem**
+A brief description of the problem you encountered while sealing a sector.
 
-A brief description of the problem you encountered while proving (sealing) a sector.
+**Version**
 
-Including what commands you ran, and a description of your setup, is very helpful.
+The output of `lotus --version`.
+
+**Setup**
+
+You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
+
+**Commands**
+
+Commands you ran.
 
 **Sectors status**
 
@@ -37,7 +48,3 @@ Please collect the following diagnostic information, and share a link here
 
 If you have modified parts of lotus, please describe which areas were modified,
 and the scope of those modifications
-
-**Version**
-
-The output of `lotus --version`.

CHANGELOG.md

@@ -1,11 +1,116 @@
 # Lotus changelog
 
+# 1.5.2 / 2021-03-11
+
+This is an hotfix release of Lotus that fixes a critical bug introduced in v1.5.1 in the miner windowPoSt logic. This upgrade is only affecting miner nodes.
+
+## Changes
+- fix window post rand check (https://github.com/filecoin-project/lotus/pull/5773)
+- wdpost: Always use head tipset to get randomness (https://github.com/filecoin-project/lotus/pull/5774)
+
+# 1.5.1 / 2021-03-10
+
+This is an optional release of Lotus that introduces an important fix to the WindowPoSt computation process. The change is to wait for some confidence before drawing beacon randomness for the proof. Without this, invalid proofs might be generated as the result of a null tipset. 
+
+## Splitstore
+
+This release also introduces the splitstore, a new optional blockstore that segregates the monolithic blockstore into cold and hot regions. The hot region contains objects from the last 4-5 finalities plus all reachable objects from two finalities away. All other objects are moved to the cold region using a compaction process that executes every finality, once 5 finalities have elapsed.
+
+The splitstore allows us to separate the two regions quite effectively, using two separate badger blockstores. The separation
+means that the live working set is much smaller, which results in potentially significant performance improvements. In addition, it means that the coldstore can be moved to a separate (bigger, slower, cheaper) disk without loss of performance.
+
+The design also allows us to use different implementations for the two blockstores; for example, an append-only blockstore could be used for coldstore and a faster memory mapped blockstore could be used for the hotstore (eg LMDB). We plan to experiment with these options in the future.
+
+Once the splitstore has been enabled, the existing monolithic blockstore becomes the coldstore. On the first head change notification, the splitstore will warm up the hotstore by copying all reachable objects from the current tipset into the hotstore.  All new writes go into the hotstore, with the splitstore tracking the write epoch. Once 5 finalities have elapsed, and every finality thereafter, the splitstore compacts by moving cold objects into the coldstore. There is also experimental support for garbage collection, whereby nunreachable objects are simply discarded.
+
+To enable the splitstore, add the following to config.toml:
+
+```
+[Chainstore]
+  EnableSplitstore = true
+```
+
+## Highlights
+
+Other highlights include:
+
+- Improved deal data handling - now multiple deals can be adding to sectors in parallel
+- Rewriten sector pledging - it now actually cares about max sealing sector limits
+- Better handling for sectors stuck in the RecoverDealIDs state
+- lotus-miner sectors extend command
+- Optional configurable storage path size limit
+- Config to disable owner/worker fallback from control addresses (useful when owner is a key on a hardware wallet)
+- A write log for node metadata, which can be restored as a backup when the metadata leveldb becomes corrupted (e.g. when you run out of disk space / system crashes in some bad way)
+
+## Changes
+
+- avoid use mp.cfg directly to avoid race (https://github.com/filecoin-project/lotus/pull/5350)
+- Show replacing message CID is state search-msg cli (https://github.com/filecoin-project/lotus/pull/5656)
+- Fix riceing by importing the main package (https://github.com/filecoin-project/lotus/pull/5675)
+- Remove sectors with all deals expired in RecoverDealIDs (https://github.com/filecoin-project/lotus/pull/5658)
+- storagefsm: Rewrite input handling (https://github.com/filecoin-project/lotus/pull/5375)
+- reintroduce Refactor send command for better testability (https://github.com/filecoin-project/lotus/pull/5668)
+- Improve error message with importing a chain (https://github.com/filecoin-project/lotus/pull/5669)
+- storagefsm: Cleanup CC sector creation (https://github.com/filecoin-project/lotus/pull/5612)
+- chain list --gas-stats display capacity (https://github.com/filecoin-project/lotus/pull/5676)
+- Correct some logs (https://github.com/filecoin-project/lotus/pull/5694)
+- refactor blockstores (https://github.com/filecoin-project/lotus/pull/5484)
+- Add idle to sync stage's String() (https://github.com/filecoin-project/lotus/pull/5702)
+- packer provisioner (https://github.com/filecoin-project/lotus/pull/5604)
+- add DeleteMany to Blockstore interface (https://github.com/filecoin-project/lotus/pull/5703)
+- segregate chain and state blockstores (https://github.com/filecoin-project/lotus/pull/5695)
+- fix(multisig): The format of the amount is not correct in msigLockApp (https://github.com/filecoin-project/lotus/pull/5718)
+- Update butterfly network (https://github.com/filecoin-project/lotus/pull/5627)
+- Collect worker task metrics (https://github.com/filecoin-project/lotus/pull/5648)
+- Correctly format disputer log (https://github.com/filecoin-project/lotus/pull/5716)
+- Log block CID in the large delay warning (https://github.com/filecoin-project/lotus/pull/5704)
+- Move api client builders to a cliutil package (https://github.com/filecoin-project/lotus/pull/5728)
+- Implement net peers --extended (https://github.com/filecoin-project/lotus/pull/5734)
+- Command to extend sector expiration (https://github.com/filecoin-project/lotus/pull/5666)
+- garbage collect hotstore after compaction (https://github.com/filecoin-project/lotus/pull/5744)
+- tune badger gc to repeatedly gc the value log until there is no rewrite (https://github.com/filecoin-project/lotus/pull/5745)
+- Add configuration option for pubsub IPColocationWhitelist subnets (https://github.com/filecoin-project/lotus/pull/5735)
+- hot/cold blockstore segregation (aka. splitstore) (https://github.com/filecoin-project/lotus/pull/4992)
+- Customize verifreg root key and remainder account when making genesis (https://github.com/filecoin-project/lotus/pull/5730)
+- chore: update go-graphsync to 0.6.0 (https://github.com/filecoin-project/lotus/pull/5746)
+- Add connmgr metadata to NetPeerInfo (https://github.com/filecoin-project/lotus/pull/5749)
+- test: attempt to make the splitstore test deterministic (https://github.com/filecoin-project/lotus/pull/5750)
+- Feat/api no dep build (https://github.com/filecoin-project/lotus/pull/5729)
+- Fix bootstrapper profile setting (https://github.com/filecoin-project/lotus/pull/5756)
+- Check liveness of sectors when processing termination batches (https://github.com/filecoin-project/lotus/pull/5759)
+- Configurable storage path storage limit (https://github.com/filecoin-project/lotus/pull/5624)
+- miner: Config to disable owner/worker address fallback (https://github.com/filecoin-project/lotus/pull/5620)
+- Fix TestUnpadReader on Go 1.16 (https://github.com/filecoin-project/lotus/pull/5761)
+- Metadata datastore log (https://github.com/filecoin-project/lotus/pull/5755)
+- Remove the SR2 stats, leave just the network totals (https://github.com/filecoin-project/lotus/pull/5757)
+- fix: wait a bit before starting to compute window post proofs (https://github.com/filecoin-project/lotus/pull/5764)
+- fix: retry proof when randomness changes (https://github.com/filecoin-project/lotus/pull/5768)
+
+
+# 1.5.0 / 2021-02-23
+
+This is a mandatory release of Lotus that introduces the fifth upgrade to the Filecoin network. The network upgrade occurs at height 550321, before which time all nodes must have updated to this release (or later). At this height, [v3 specs-actors](https://github.com/filecoin-project/specs-actors/releases/tag/v3.0.0) will take effect, which in turn implements the following two FIPs:
+
+- [FIP-0007 h/amt-v3](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0007.md) which improves the performance of the Filecoin HAMT and AMT.
+- [FIP-0010 off-chain Window PoSt Verification](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0010.md) which reduces the gas consumption of `SubmitWindowedPoSt` messages significantly by optimistically accepting Window PoSt proofs without verification, and allowing them to be disputed later by off-chain verifiers.
+
+Note that the integration of v3 actors was already completed in 1.4.2, this upgrade simply sets the epoch for the upgrade to occur.
+
+## Disputer
+
+FIP-0010 introduces the ability to dispute bad Window PoSts. Node operators are encouraged to run the new Lotus disputer alongside their Lotus daemons. For more information, see the announcement [here](https://github.com/filecoin-project/lotus/discussions/5617#discussioncomment-387333).
+
+## Changes
+
+- [#5341](https://github.com/filecoin-project/lotus/pull/5341)  Add a  `LOTUS_DISABLE_V3_ACTOR_MIGRATION` envvar
+    - Setting this envvar to 1 disables the v3 actor migration, should only be used in the event of a failed migration
+
 # 1.4.2 / 2021-02-17
 
 This is a large, and highly recommended, optional release with new features and improvements for lotus miner and deal-making UX. The release also integrates [v3 specs-actors](https://github.com/filecoin-project/specs-actors/releases/tag/v3.0.0), which implements two FIPs:
 
 - [FIP-0007 h/amt-v3](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0007.md) which improves the performance of the Filecoin HAMT and AMT.
-- [FIP-0010 off-chain Window PoSt Verification](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0010.md) which reduces the gas consumption of `SubmitedWindowedPost` messages significantly by optimistically accepting Window PoSt proofs without verification, and allowing them to be disputed later by off-chain verifiers.
+- [FIP-0010 off-chain Window PoSt Verification](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0010.md) which reduces the gas consumption of `SubmitWindowedPoSt` messages significantly by optimistically accepting Window PoSt proofs without verification, and allowing them to be disputed later by off-chain verifiers.
  
 Note that this release does NOT set an upgrade epoch for v3 actors to take effect. That will be done in the upcoming 1.5.0 release.
  

Makefile

@@ -75,6 +75,9 @@ calibnet: lotus lotus-miner lotus-worker lotus-seed
 nerpanet: GOFLAGS+=-tags=nerpanet
 nerpanet: lotus lotus-miner lotus-worker lotus-seed
 
+butterflynet: GOFLAGS+=-tags=butterflynet
+butterflynet: lotus lotus-miner lotus-worker lotus-seed
+
 lotus: $(BUILD_DEPS)
 	rm -f lotus
 	go build $(GOFLAGS) -o lotus ./cmd/lotus

api/api_common.go

@@ -11,8 +11,6 @@ import (
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	protocol "github.com/libp2p/go-libp2p-core/protocol"
-
-	"github.com/filecoin-project/lotus/build"
 )
 
 type Common interface {
@@ -33,6 +31,7 @@ type Common interface {
 	NetPubsubScores(context.Context) ([]PubsubScore, error)
 	NetAutoNatStatus(context.Context) (NatInfo, error)
 	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)
+	NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error)
 
 	// NetBandwidthStats returns statistics about the nodes total bandwidth
 	// usage and current rate across all peers and protocols.
@@ -57,7 +56,7 @@ type Common interface {
 	ID(context.Context) (peer.ID, error)
 
 	// Version provides information about API provider
-	Version(context.Context) (Version, error)
+	Version(context.Context) (APIVersion, error)
 
 	LogList(context.Context) ([]string, error)
 	LogSetLevel(context.Context, string, string) error
@@ -71,15 +70,15 @@ type Common interface {
 	Closing(context.Context) (<-chan struct{}, error)
 }
 
-// Version provides various build-time information
+// APIVersion provides various build-time information
-type Version struct {
+type APIVersion struct {
 	Version string
 
 	// APIVersion is a binary encoded semver version of the remote implementing
 	// this api
 	//
 	// See APIVersion in build/version.go
-	APIVersion build.Version
+	APIVersion Version
 
 	// TODO: git commit / os / genesis cid?
 
@@ -87,7 +86,7 @@ type Version struct {
 	BlockDelay uint64
 }
 
-func (v Version) String() string {
+func (v APIVersion) String() string {
 	return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
 }
 

api/api_full.go

@@ -32,6 +32,14 @@ import (
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
+//go:generate go run github.com/golang/mock/mockgen -destination=mocks/mock_full.go -package=mocks . FullNode
+
+// ChainIO abstracts operations for accessing raw IPLD objects.
+type ChainIO interface {
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainHasObj(context.Context, cid.Cid) (bool, error)
+}
+
 // FullNode API is a low-level interface to the Filecoin network full node
 type FullNode interface {
 	Common
@@ -862,6 +870,8 @@ const (
 
 func (v SyncStateStage) String() string {
 	switch v {
+	case StageIdle:
+		return "idle"
 	case StageHeaders:
 		return "header sync"
 	case StagePersistHeaders:

api/api_storage.go

@@ -36,7 +36,7 @@ type StorageMiner interface {
 	MiningBase(context.Context) (*types.TipSet, error)
 
 	// Temp api for testing
-	PledgeSector(context.Context) error
+	PledgeSector(context.Context) (abi.SectorID, error)
 
 	// Get the status of a given sector by ID
 	SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error)
@@ -238,6 +238,9 @@ type AddressConfig struct {
 	PreCommitControl []address.Address
 	CommitControl    []address.Address
 	TerminateControl []address.Address
+
+	DisableOwnerFallback  bool
+	DisableWorkerFallback bool
 }
 
 // PendingDealInfo has info about pending deals and when they are due to be

api/api_worker.go

@@ -9,12 +9,10 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-
-	"github.com/filecoin-project/lotus/build"
 )
 
 type WorkerAPI interface {
-	Version(context.Context) (build.Version, error)
+	Version(context.Context) (Version, error)
 	// TODO: Info() (name, ...) ?
 
 	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // TaskType -> Weight

api/apibstore/apibstore.go

@@ -1,68 +0,0 @@
-package apibstore
-
-import (
-	"context"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/lib/blockstore"
-)
-
-type ChainIO interface {
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-}
-
-type apiBStore struct {
-	api ChainIO
-}
-
-func NewAPIBlockstore(cio ChainIO) blockstore.Blockstore {
-	return &apiBStore{
-		api: cio,
-	}
-}
-
-func (a *apiBStore) DeleteBlock(cid.Cid) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBStore) Has(c cid.Cid) (bool, error) {
-	return a.api.ChainHasObj(context.TODO(), c)
-}
-
-func (a *apiBStore) Get(c cid.Cid) (blocks.Block, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return nil, err
-	}
-	return blocks.NewBlockWithCid(bb, c)
-}
-
-func (a *apiBStore) GetSize(c cid.Cid) (int, error) {
-	bb, err := a.api.ChainReadObj(context.TODO(), c)
-	if err != nil {
-		return 0, err
-	}
-	return len(bb), nil
-}
-
-func (a *apiBStore) Put(blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBStore) PutMany([]blocks.Block) error {
-	return xerrors.New("not supported")
-}
-
-func (a *apiBStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.New("not supported")
-}
-
-func (a *apiBStore) HashOnRead(enabled bool) {
-	return
-}
-
-var _ blockstore.Blockstore = &apiBStore{}

api/apistruct/struct.go

@@ -33,7 +33,6 @@ import (
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -60,12 +59,13 @@ type CommonStruct struct {
 		NetBandwidthStatsByPeer     func(ctx context.Context) (map[string]metrics.Stats, error)      `perm:"read"`
 		NetBandwidthStatsByProtocol func(ctx context.Context) (map[protocol.ID]metrics.Stats, error) `perm:"read"`
 		NetAgentVersion             func(ctx context.Context, p peer.ID) (string, error)             `perm:"read"`
+		NetPeerInfo                 func(context.Context, peer.ID) (*api.ExtendedPeerInfo, error)    `perm:"read"`
 		NetBlockAdd                 func(ctx context.Context, acl api.NetBlockList) error            `perm:"admin"`
 		NetBlockRemove              func(ctx context.Context, acl api.NetBlockList) error            `perm:"admin"`
 		NetBlockList                func(ctx context.Context) (api.NetBlockList, error)              `perm:"read"`
 
-		ID      func(context.Context) (peer.ID, error)     `perm:"read"`
+		ID      func(context.Context) (peer.ID, error)        `perm:"read"`
-		Version func(context.Context) (api.Version, error) `perm:"read"`
+		Version func(context.Context) (api.APIVersion, error) `perm:"read"`
 
 		LogList     func(context.Context) ([]string, error)     `perm:"write"`
 		LogSetLevel func(context.Context, string, string) error `perm:"write"`
@@ -304,7 +304,7 @@ type StorageMinerStruct struct {
 		MarketPendingDeals        func(ctx context.Context) (api.PendingDealInfo, error)                                                                                                                       `perm:"write"`
 		MarketPublishPendingDeals func(ctx context.Context) error                                                                                                                                              `perm:"admin"`
 
-		PledgeSector func(context.Context) error `perm:"write"`
+		PledgeSector func(context.Context) (abi.SectorID, error) `perm:"write"`
 
 		SectorsStatus                 func(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) `perm:"read"`
 		SectorsList                   func(context.Context) ([]abi.SectorNumber, error)                                             `perm:"read"`
@@ -389,7 +389,7 @@ type WorkerStruct struct {
 	Internal struct {
 		// TODO: lower perms
 
-		Version func(context.Context) (build.Version, error) `perm:"admin"`
+		Version func(context.Context) (api.Version, error) `perm:"admin"`
 
 		TaskTypes func(context.Context) (map[sealtasks.TaskType]struct{}, error) `perm:"admin"`
 		Paths     func(context.Context) ([]stores.StoragePath, error)            `perm:"admin"`
@@ -540,13 +540,17 @@ func (c *CommonStruct) NetAgentVersion(ctx context.Context, p peer.ID) (string,
 	return c.Internal.NetAgentVersion(ctx, p)
 }
 
+func (c *CommonStruct) NetPeerInfo(ctx context.Context, p peer.ID) (*api.ExtendedPeerInfo, error) {
+	return c.Internal.NetPeerInfo(ctx, p)
+}
+
 // ID implements API.ID
 func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
 	return c.Internal.ID(ctx)
 }
 
 // Version implements API.Version
-func (c *CommonStruct) Version(ctx context.Context) (api.Version, error) {
+func (c *CommonStruct) Version(ctx context.Context) (api.APIVersion, error) {
 	return c.Internal.Version(ctx)
 }
 
@@ -1274,7 +1278,7 @@ func (c *StorageMinerStruct) ActorAddressConfig(ctx context.Context) (api.Addres
 	return c.Internal.ActorAddressConfig(ctx)
 }
 
-func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
+func (c *StorageMinerStruct) PledgeSector(ctx context.Context) (abi.SectorID, error) {
 	return c.Internal.PledgeSector(ctx)
 }
 
@@ -1610,7 +1614,7 @@ func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.Registere
 
 // WorkerStruct
 
-func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {
+func (w *WorkerStruct) Version(ctx context.Context) (api.Version, error) {
 	return w.Internal.Version(ctx)
 }
 

api/cbor_gen.go

@@ -5,14 +5,18 @@ package api
 import (
 	"fmt"
 	"io"
+	"sort"
 
 	abi "github.com/filecoin-project/go-state-types/abi"
 	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
+	cid "github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )
 
 var _ = xerrors.Errorf
+var _ = cid.Undef
+var _ = sort.Sort
 
 func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	if t == nil {
@@ -171,7 +175,8 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
+			// Field doesn't exist on this type, so ignore it
+			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 
@@ -319,7 +324,8 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
+			// Field doesn't exist on this type, so ignore it
+			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 
@@ -427,7 +433,8 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
+			// Field doesn't exist on this type, so ignore it
+			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 
@@ -575,7 +582,8 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
+			// Field doesn't exist on this type, so ignore it
+			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 
@@ -723,7 +731,8 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 			}
 
 		default:
-			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
+			// Field doesn't exist on this type, so ignore it
+			cbg.ScanForLinks(r, func(cid.Cid) {})
 		}
 	}
 

api/docgen/docgen.go

@@ -113,7 +113,7 @@ func init() {
 	addExample(network.Connected)
 	addExample(dtypes.NetworkName("lotus"))
 	addExample(api.SyncStateStage(1))
-	addExample(build.FullAPIVersion)
+	addExample(api.FullAPIVersion)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
 	addExample(datatransfer.TransferID(3))
@@ -123,6 +123,8 @@ func init() {
 	addExample(retrievalmarket.DealStatusNew)
 	addExample(network.ReachabilityPublic)
 	addExample(build.NewestNetworkVersion)
+	addExample(map[string]int{"name": 42})
+	addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
 	addExample(&types.ExecutionTrace{
 		Msg:    exampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
 		MsgRct: exampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),

api/test/paych.go

@@ -15,7 +15,7 @@ import (
 	cbor "github.com/ipfs/go-ipld-cbor"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -132,7 +132,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
 		t.Fatal("Unable to settle payment channel")
 	}
 
-	creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(paymentCreator)))
+	creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(paymentCreator)))
 
 	// wait for the receiver to submit their vouchers
 	ev := events.NewEvents(ctx, paymentCreator)

api/test/tape.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
@@ -18,7 +17,9 @@ import (
 )
 
 func TestTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration) {
-	t.Run("before", func(t *testing.T) { testTapeFix(t, b, blocktime, false) })
+	// The "before" case is disabled, because we need the builder to mock 32 GiB sectors to accurately repro this case
+	// TODO: Make the mock sector size configurable and reenable this
+	//t.Run("before", func(t *testing.T) { testTapeFix(t, b, blocktime, false) })
 	t.Run("after", func(t *testing.T) { testTapeFix(t, b, blocktime, true) })
 }
 func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool) {
@@ -73,23 +74,9 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool
 		<-done
 	}()
 
-	err = miner.PledgeSector(ctx)
+	sid, err := miner.PledgeSector(ctx)
 	require.NoError(t, err)
 
-	// Wait till done.
-	var sectorNo abi.SectorNumber
-	for {
-		s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM
-		require.NoError(t, err)
-		fmt.Printf("Sectors: %d\n", len(s))
-		if len(s) == 1 {
-			sectorNo = s[0]
-			break
-		}
-
-		build.Clock.Sleep(100 * time.Millisecond)
-	}
-
 	fmt.Printf("All sectors is fsm\n")
 
 	// If before, we expect the precommit to fail
@@ -101,7 +88,7 @@ func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool
 	}
 
 	for {
-		st, err := miner.SectorsStatus(ctx, sectorNo, false)
+		st, err := miner.SectorsStatus(ctx, sid.Number, false)
 		require.NoError(t, err)
 		if st.State == successState {
 			break

api/test/test.go

@@ -155,13 +155,13 @@ var MineNext = miner.MineReq{
 }
 
 func (ts *testSuite) testVersion(t *testing.T) {
-	build.RunningNodeType = build.NodeFull
+	api.RunningNodeType = api.NodeFull
 
 	ctx := context.Background()
 	apis, _ := ts.makeNodes(t, OneFull, OneMiner)
-	api := apis[0]
+	napi := apis[0]
 
-	v, err := api.Version(ctx)
+	v, err := napi.Version(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}

api/test/window_post.go

@@ -162,7 +162,7 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n,
 			log.Errorf("WAIT")
 		}
 		log.Errorf("PLEDGING %d", i)
-		err := miner.PledgeSector(ctx)
+		_, err := miner.PledgeSector(ctx)
 		require.NoError(t, err)
 	}
 

api/types.go

@@ -3,6 +3,7 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -99,3 +100,18 @@ type NetBlockList struct {
 	IPAddrs   []string
 	IPSubnets []string
 }
+
+type ExtendedPeerInfo struct {
+	ID          peer.ID
+	Agent       string
+	Addrs       []string
+	Protocols   []string
+	ConnMgrMeta *ConnMgrInfo
+}
+
+type ConnMgrInfo struct {
+	FirstSeen time.Time
+	Value     int
+	Tags      map[string]int
+	Conns     map[string]time.Time
+}

api/version.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"fmt"
+
+	xerrors "golang.org/x/xerrors"
+)
+
+type Version uint32
+
+func newVer(major, minor, patch uint8) Version {
+	return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
+}
+
+// Ints returns (major, minor, patch) versions
+func (ve Version) Ints() (uint32, uint32, uint32) {
+	v := uint32(ve)
+	return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
+}
+
+func (ve Version) String() string {
+	vmj, vmi, vp := ve.Ints()
+	return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
+}
+
+func (ve Version) EqMajorMinor(v2 Version) bool {
+	return ve&minorMask == v2&minorMask
+}
+
+type NodeType int
+
+const (
+	NodeUnknown NodeType = iota
+
+	NodeFull
+	NodeMiner
+	NodeWorker
+)
+
+var RunningNodeType NodeType
+
+func VersionForType(nodeType NodeType) (Version, error) {
+	switch nodeType {
+	case NodeFull:
+		return FullAPIVersion, nil
+	case NodeMiner:
+		return MinerAPIVersion, nil
+	case NodeWorker:
+		return WorkerAPIVersion, nil
+	default:
+		return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
+	}
+}
+
+// semver versions of the rpc api exposed
+var (
+	FullAPIVersion   = newVer(1, 1, 0)
+	MinerAPIVersion  = newVer(1, 0, 1)
+	WorkerAPIVersion = newVer(1, 0, 0)
+)
+
+//nolint:varcheck,deadcode
+const (
+	majorMask = 0xff0000
+	minorMask = 0xffff00
+	patchMask = 0xffffff
+
+	majorOnlyMask = 0xff0000
+	minorOnlyMask = 0x00ff00
+	patchOnlyMask = 0x0000ff
+)

blockstore/api.go

@@ -0,0 +1,66 @@
+package blockstore
+
+import (
+	"context"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
+)
+
+type ChainIO interface {
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainHasObj(context.Context, cid.Cid) (bool, error)
+}
+
+type apiBlockstore struct {
+	api ChainIO
+}
+
+// This blockstore is adapted in the constructor.
+var _ BasicBlockstore = (*apiBlockstore)(nil)
+
+func NewAPIBlockstore(cio ChainIO) Blockstore {
+	bs := &apiBlockstore{api: cio}
+	return Adapt(bs) // return an adapted blockstore.
+}
+
+func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
+	return a.api.ChainHasObj(context.TODO(), c)
+}
+
+func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return nil, err
+	}
+	return blocks.NewBlockWithCid(bb, c)
+}
+
+func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
+	bb, err := a.api.ChainReadObj(context.TODO(), c)
+	if err != nil {
+		return 0, err
+	}
+	return len(bb), nil
+}
+
+func (a *apiBlockstore) Put(blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBlockstore) PutMany([]blocks.Block) error {
+	return xerrors.New("not supported")
+}
+
+func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	return nil, xerrors.New("not supported")
+}
+
+func (a *apiBlockstore) HashOnRead(enabled bool) {
+	return
+}

blockstore/badger/blockstore.go

@@ -16,7 +16,7 @@ import (
 	logger "github.com/ipfs/go-log/v2"
 	pool "github.com/libp2p/go-buffer-pool"
 
-	"github.com/filecoin-project/lotus/lib/blockstore"
+	"github.com/filecoin-project/lotus/blockstore"
 )
 
 var (
@@ -110,10 +110,7 @@ func Open(opts Options) (*Blockstore, error) {
 		return nil, fmt.Errorf("failed to open badger blockstore: %w", err)
 	}
 
-	bs := &Blockstore{
+	bs := &Blockstore{DB: db}
-		DB: db,
-	}
-
 	if p := opts.Prefix; p != "" {
 		bs.prefixing = true
 		bs.prefix = []byte(p)
@@ -134,6 +131,25 @@ func (b *Blockstore) Close() error {
 	return b.DB.Close()
 }
 
+// CollectGarbage runs garbage collection on the value log
+func (b *Blockstore) CollectGarbage() error {
+	if atomic.LoadInt64(&b.state) != stateOpen {
+		return ErrBlockstoreClosed
+	}
+
+	var err error
+	for err == nil {
+		err = b.DB.RunValueLogGC(0.125)
+	}
+
+	if err == badger.ErrNoRewrite {
+		// not really an error in this case
+		return nil
+	}
+
+	return err
+}
+
 // View implements blockstore.Viewer, which leverages zero-copy read-only
 // access to values.
 func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
@@ -321,6 +337,44 @@ func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
 	})
 }
 
+func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
+	if atomic.LoadInt64(&b.state) != stateOpen {
+		return ErrBlockstoreClosed
+	}
+
+	batch := b.DB.NewWriteBatch()
+	defer batch.Cancel()
+
+	// toReturn tracks the byte slices to return to the pool, if we're using key
+	// prefixing. we can't return each slice to the pool after each Set, because
+	// badger holds on to the slice.
+	var toReturn [][]byte
+	if b.prefixing {
+		toReturn = make([][]byte, 0, len(cids))
+		defer func() {
+			for _, b := range toReturn {
+				KeyPool.Put(b)
+			}
+		}()
+	}
+
+	for _, cid := range cids {
+		k, pooled := b.PooledStorageKey(cid)
+		if pooled {
+			toReturn = append(toReturn, k)
+		}
+		if err := batch.Delete(k); err != nil {
+			return err
+		}
+	}
+
+	err := batch.Flush()
+	if err != nil {
+		err = fmt.Errorf("failed to delete blocks from badger blockstore: %w", err)
+	}
+	return err
+}
+
 // AllKeysChan implements Blockstore.AllKeysChan.
 func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	if atomic.LoadInt64(&b.state) != stateOpen {

blockstore/badger/blockstore_test.go

@@ -6,8 +6,9 @@ import (
 	"testing"
 
 	blocks "github.com/ipfs/go-block-format"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/blockstore"
 )
 
 func TestBadgerBlockstore(t *testing.T) {
@@ -60,8 +61,8 @@ func TestStorageKey(t *testing.T) {
 	require.Equal(t, k3, k2)
 }
 
-func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.Blockstore, path string) {
+func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
-	return func(tb testing.TB) (bs blockstore.Blockstore, path string) {
+	return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
 		tb.Helper()
 
 		path, err := ioutil.TempDir("", "")
@@ -82,8 +83,8 @@ func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (
 	}
 }
 
-func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.Blockstore, err error) {
+func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
-	return func(tb testing.TB, path string) (bs blockstore.Blockstore, err error) {
+	return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) {
 		tb.Helper()
 		return Open(optsSupplier(path))
 	}

blockstore/badger/blockstore_test_suite.go

@@ -8,18 +8,19 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	u "github.com/ipfs/go-ipfs-util"
 
+	"github.com/filecoin-project/lotus/blockstore"
+
 	"github.com/stretchr/testify/require"
 )
 
 // TODO: move this to go-ipfs-blockstore.
 type Suite struct {
-	NewBlockstore  func(tb testing.TB) (bs blockstore.Blockstore, path string)
+	NewBlockstore  func(tb testing.TB) (bs blockstore.BasicBlockstore, path string)
-	OpenBlockstore func(tb testing.TB, path string) (bs blockstore.Blockstore, err error)
+	OpenBlockstore func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error)
 }
 
 func (s *Suite) RunTests(t *testing.T, prefix string) {
@@ -290,7 +291,7 @@ func (s *Suite) TestDelete(t *testing.T) {
 
 }
 
-func insertBlocks(t *testing.T, bs blockstore.Blockstore, count int) []cid.Cid {
+func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
 	keys := make([]cid.Cid, count)
 	for i := 0; i < count; i++ {
 		block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))

blockstore/blockstore.go

@@ -0,0 +1,95 @@
+package blockstore
+
+import (
+	cid "github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
+	logging "github.com/ipfs/go-log/v2"
+
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+)
+
+var log = logging.Logger("blockstore")
+
+var ErrNotFound = blockstore.ErrNotFound
+
+// Blockstore is the blockstore interface used by Lotus. It is the union
+// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
+// e.g. View or Sync.
+type Blockstore interface {
+	blockstore.Blockstore
+	blockstore.Viewer
+	BatchDeleter
+}
+
+// BasicBlockstore is an alias to the original IPFS Blockstore.
+type BasicBlockstore = blockstore.Blockstore
+
+type Viewer = blockstore.Viewer
+
+type BatchDeleter interface {
+	DeleteMany(cids []cid.Cid) error
+}
+
+// WrapIDStore wraps the underlying blockstore in an "identity" blockstore.
+// The ID store filters out all puts for blocks with CIDs using the "identity"
+// hash function. It also extracts inlined blocks from CIDs using the identity
+// hash function and returns them on get/has, ignoring the contents of the
+// blockstore.
+func WrapIDStore(bstore blockstore.Blockstore) Blockstore {
+	if is, ok := bstore.(*idstore); ok {
+		// already wrapped
+		return is
+	}
+
+	if bs, ok := bstore.(Blockstore); ok {
+		// we need to wrap our own because we don't want to neuter the DeleteMany method
+		// the underlying blockstore has implemented an (efficient) DeleteMany
+		return NewIDStore(bs)
+	}
+
+	// The underlying blockstore does not implement DeleteMany, so we need to shim it.
+	// This is less efficient as it'll iterate and perform single deletes.
+	return NewIDStore(Adapt(bstore))
+}
+
+// FromDatastore creates a new blockstore backed by the given datastore.
+func FromDatastore(dstore ds.Batching) Blockstore {
+	return WrapIDStore(blockstore.NewBlockstore(dstore))
+}
+
+type adaptedBlockstore struct {
+	blockstore.Blockstore
+}
+
+var _ Blockstore = (*adaptedBlockstore)(nil)
+
+func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
+	blk, err := a.Get(cid)
+	if err != nil {
+		return err
+	}
+	return callback(blk.RawData())
+}
+
+func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
+	for _, cid := range cids {
+		err := a.DeleteBlock(cid)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Adapt adapts a standard blockstore to a Lotus blockstore by
+// enriching it with the extra methods that Lotus requires (e.g. View, Sync).
+//
+// View proxies over to Get and calls the callback with the value supplied by Get.
+// Sync noops.
+func Adapt(bs blockstore.Blockstore) Blockstore {
+	if ret, ok := bs.(Blockstore); ok {
+		return ret
+	}
+	return &adaptedBlockstore{bs}
+}

blockstore/buffered.go

@@ -0,0 +1,174 @@
+package blockstore
+
+import (
+	"context"
+	"os"
+
+	block "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+)
+
+// buflog is a logger for the buffered blockstore. It is subscoped from the
+// blockstore logger.
+var buflog = log.Named("buf")
+
+type BufferedBlockstore struct {
+	read  Blockstore
+	write Blockstore
+}
+
+func NewBuffered(base Blockstore) *BufferedBlockstore {
+	var buf Blockstore
+	if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" {
+		buflog.Warn("VM BLOCKSTORE BUFFERING IS DISABLED")
+		buf = base
+	} else {
+		buf = NewMemory()
+	}
+
+	bs := &BufferedBlockstore{
+		read:  base,
+		write: buf,
+	}
+	return bs
+}
+
+func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBlockstore {
+	return &BufferedBlockstore{
+		read:  r,
+		write: w,
+	}
+}
+
+var (
+	_ Blockstore = (*BufferedBlockstore)(nil)
+	_ Viewer     = (*BufferedBlockstore)(nil)
+)
+
+func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	a, err := bs.read.AllKeysChan(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := bs.write.AllKeysChan(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make(chan cid.Cid)
+	go func() {
+		defer close(out)
+		for a != nil || b != nil {
+			select {
+			case val, ok := <-a:
+				if !ok {
+					a = nil
+				} else {
+					select {
+					case out <- val:
+					case <-ctx.Done():
+						return
+					}
+				}
+			case val, ok := <-b:
+				if !ok {
+					b = nil
+				} else {
+					select {
+					case out <- val:
+					case <-ctx.Done():
+						return
+					}
+				}
+			}
+		}
+	}()
+
+	return out, nil
+}
+
+func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
+	if err := bs.read.DeleteBlock(c); err != nil {
+		return err
+	}
+
+	return bs.write.DeleteBlock(c)
+}
+
+func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
+	if err := bs.read.DeleteMany(cids); err != nil {
+		return err
+	}
+
+	return bs.write.DeleteMany(cids)
+}
+
+func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
+	// both stores are viewable.
+	if err := bs.write.View(c, callback); err == ErrNotFound {
+		// not found in write blockstore; fall through.
+	} else {
+		return err // propagate errors, or nil, i.e. found.
+	}
+	return bs.read.View(c, callback)
+}
+
+func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
+	if out, err := bs.write.Get(c); err != nil {
+		if err != ErrNotFound {
+			return nil, err
+		}
+	} else {
+		return out, nil
+	}
+
+	return bs.read.Get(c)
+}
+
+func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
+	s, err := bs.read.GetSize(c)
+	if err == ErrNotFound || s == 0 {
+		return bs.write.GetSize(c)
+	}
+
+	return s, err
+}
+
+func (bs *BufferedBlockstore) Put(blk block.Block) error {
+	has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
+	if err != nil {
+		return err
+	}
+
+	if has {
+		return nil
+	}
+
+	return bs.write.Put(blk)
+}
+
+func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
+	has, err := bs.write.Has(c)
+	if err != nil {
+		return false, err
+	}
+	if has {
+		return true, nil
+	}
+
+	return bs.read.Has(c)
+}
+
+func (bs *BufferedBlockstore) HashOnRead(hor bool) {
+	bs.read.HashOnRead(hor)
+	bs.write.HashOnRead(hor)
+}
+
+func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
+	return bs.write.PutMany(blks)
+}
+
+func (bs *BufferedBlockstore) Read() Blockstore {
+	return bs.read
+}

blockstore/cached.go

@@ -0,0 +1,25 @@
+package blockstore
+
+import (
+	"context"
+
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+)
+
+type CacheOpts = blockstore.CacheOpts
+
+func DefaultCacheOpts() CacheOpts {
+	return CacheOpts{
+		HasBloomFilterSize:   0,
+		HasBloomFilterHashes: 0,
+		HasARCCacheSize:      512 << 10,
+	}
+}
+
+func CachedBlockstore(ctx context.Context, bs Blockstore, opts CacheOpts) (Blockstore, error) {
+	cached, err := blockstore.CachedBlockstore(ctx, bs, opts)
+	if err != nil {
+		return nil, err
+	}
+	return WrapIDStore(cached), nil
+}

blockstore/doc.go

@@ -0,0 +1,9 @@
+// Package blockstore and subpackages contain most of the blockstore
+// implementations used by Lotus.
+//
+// Blockstores not ultimately constructed out of the building blocks in this
+// package may not work properly.
+//
+// This package re-exports parts of the go-ipfs-blockstore package such that
+// no other package needs to import it directly, for ergonomics and traceability.
+package blockstore

blockstore/fallback.go

@@ -9,48 +9,61 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
 )
 
-var log = logging.Logger("blockstore")
+// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
-
+// if it was a FallbackStore. Otherwise, it just returns the supplied store
-type FallbackStore struct {
+// unmodified.
-	blockstore.Blockstore
+func UnwrapFallbackStore(bs Blockstore) (Blockstore, bool) {
-
+	if fbs, ok := bs.(*FallbackStore); ok {
-	fallbackGetBlock func(context.Context, cid.Cid) (blocks.Block, error)
+		return fbs.Blockstore, true
-	lk               sync.RWMutex
+	}
+	return bs, false
 }
 
-func (fbs *FallbackStore) SetFallback(fg func(context.Context, cid.Cid) (blocks.Block, error)) {
+// FallbackStore is a read-through store that queries another (potentially
+// remote) source if the block is not found locally. If the block is found
+// during the fallback, it stores it in the local store.
+type FallbackStore struct {
+	Blockstore
+
+	lk sync.RWMutex
+	// missFn is the function that will be invoked on a local miss to pull the
+	// block from elsewhere.
+	missFn func(context.Context, cid.Cid) (blocks.Block, error)
+}
+
+var _ Blockstore = (*FallbackStore)(nil)
+
+func (fbs *FallbackStore) SetFallback(missFn func(context.Context, cid.Cid) (blocks.Block, error)) {
 	fbs.lk.Lock()
 	defer fbs.lk.Unlock()
 
-	fbs.fallbackGetBlock = fg
+	fbs.missFn = missFn
 }
 
 func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
-	log.Errorw("fallbackstore: Block not found locally, fetching from the network", "cid", c)
+	log.Warnf("fallbackstore: block not found locally, fetching from the network; cid: %s", c)
 	fbs.lk.RLock()
 	defer fbs.lk.RUnlock()
 
-	if fbs.fallbackGetBlock == nil {
+	if fbs.missFn == nil {
 		// FallbackStore wasn't configured yet (chainstore/bitswap aren't up yet)
 		// Wait for a bit and retry
 		fbs.lk.RUnlock()
 		time.Sleep(5 * time.Second)
 		fbs.lk.RLock()
 
-		if fbs.fallbackGetBlock == nil {
+		if fbs.missFn == nil {
-			log.Errorw("fallbackstore: fallbackGetBlock not configured yet")
+			log.Errorw("fallbackstore: missFn not configured yet")
-			return nil, blockstore.ErrNotFound
+			return nil, ErrNotFound
 		}
 	}
 
 	ctx, cancel := context.WithTimeout(context.TODO(), 120*time.Second)
 	defer cancel()
 
-	b, err := fbs.fallbackGetBlock(ctx, c)
+	b, err := fbs.missFn(ctx, c)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +82,7 @@ func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
 	switch err {
 	case nil:
 		return b, nil
-	case blockstore.ErrNotFound:
+	case ErrNotFound:
 		return fbs.getFallback(c)
 	default:
 		return b, err
@@ -81,7 +94,7 @@ func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
 	switch err {
 	case nil:
 		return sz, nil
-	case blockstore.ErrNotFound:
+	case ErrNotFound:
 		b, err := fbs.getFallback(c)
 		if err != nil {
 			return 0, err
@@ -91,5 +104,3 @@ func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
 		return sz, err
 	}
 }
-
-var _ blockstore.Blockstore = &FallbackStore{}

blockstore/idstore.go

@@ -0,0 +1,174 @@
+package blockstore
+
+import (
+	"context"
+	"io"
+
+	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+var _ Blockstore = (*idstore)(nil)
+
+type idstore struct {
+	bs Blockstore
+}
+
+func NewIDStore(bs Blockstore) Blockstore {
+	return &idstore{bs: bs}
+}
+
+func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
+	if cid.Prefix().MhType != mh.IDENTITY {
+		return false, nil, nil
+	}
+
+	dmh, err := mh.Decode(cid.Hash())
+	if err != nil {
+		return false, nil, err
+	}
+
+	if dmh.Code == mh.IDENTITY {
+		return true, dmh.Digest, nil
+	}
+
+	return false, nil, err
+}
+
+func (b *idstore) Has(cid cid.Cid) (bool, error) {
+	inline, _, err := decodeCid(cid)
+	if err != nil {
+		return false, xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return true, nil
+	}
+
+	return b.bs.Has(cid)
+}
+
+func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
+	inline, data, err := decodeCid(cid)
+	if err != nil {
+		return nil, xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return blocks.NewBlockWithCid(data, cid)
+	}
+
+	return b.bs.Get(cid)
+}
+
+func (b *idstore) GetSize(cid cid.Cid) (int, error) {
+	inline, data, err := decodeCid(cid)
+	if err != nil {
+		return 0, xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return len(data), err
+	}
+
+	return b.bs.GetSize(cid)
+}
+
+func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
+	inline, data, err := decodeCid(cid)
+	if err != nil {
+		return xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return cb(data)
+	}
+
+	return b.bs.View(cid, cb)
+}
+
+func (b *idstore) Put(blk blocks.Block) error {
+	inline, _, err := decodeCid(blk.Cid())
+	if err != nil {
+		return xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return nil
+	}
+
+	return b.bs.Put(blk)
+}
+
+func (b *idstore) PutMany(blks []blocks.Block) error {
+	toPut := make([]blocks.Block, 0, len(blks))
+	for _, blk := range blks {
+		inline, _, err := decodeCid(blk.Cid())
+		if err != nil {
+			return xerrors.Errorf("error decoding Cid: %w", err)
+		}
+
+		if inline {
+			continue
+		}
+		toPut = append(toPut, blk)
+	}
+
+	if len(toPut) > 0 {
+		return b.bs.PutMany(toPut)
+	}
+
+	return nil
+}
+
+func (b *idstore) DeleteBlock(cid cid.Cid) error {
+	inline, _, err := decodeCid(cid)
+	if err != nil {
+		return xerrors.Errorf("error decoding Cid: %w", err)
+	}
+
+	if inline {
+		return nil
+	}
+
+	return b.bs.DeleteBlock(cid)
+}
+
+func (b *idstore) DeleteMany(cids []cid.Cid) error {
+	toDelete := make([]cid.Cid, 0, len(cids))
+	for _, cid := range cids {
+		inline, _, err := decodeCid(cid)
+		if err != nil {
+			return xerrors.Errorf("error decoding Cid: %w", err)
+		}
+
+		if inline {
+			continue
+		}
+		toDelete = append(toDelete, cid)
+	}
+
+	if len(toDelete) > 0 {
+		return b.bs.DeleteMany(toDelete)
+	}
+
+	return nil
+}
+
+func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	return b.bs.AllKeysChan(ctx)
+}
+
+func (b *idstore) HashOnRead(enabled bool) {
+	b.bs.HashOnRead(enabled)
+}
+
+func (b *idstore) Close() error {
+	if c, ok := b.bs.(io.Closer); ok {
+		return c.Close()
+	}
+	return nil
+}

blockstore/ipfs.go

@@ -1,4 +1,4 @@
-package ipfsbstore
+package blockstore
 
 import (
 	"bytes"
@@ -16,16 +16,16 @@ import (
 	iface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/ipfs/interface-go-ipfs-core/options"
 	"github.com/ipfs/interface-go-ipfs-core/path"
-
-	"github.com/filecoin-project/lotus/lib/blockstore"
 )
 
-type IpfsBstore struct {
+type IPFSBlockstore struct {
 	ctx             context.Context
 	api, offlineAPI iface.CoreAPI
 }
 
-func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
+var _ BasicBlockstore = (*IPFSBlockstore)(nil)
+
+func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error) {
 	localApi, err := httpapi.NewLocalApi()
 	if err != nil {
 		return nil, xerrors.Errorf("getting local ipfs api: %w", err)
@@ -34,6 +34,7 @@ func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
 	if err != nil {
 		return nil, xerrors.Errorf("setting offline mode: %s", err)
 	}
+
 	offlineAPI := api
 	if onlineMode {
 		offlineAPI, err = localApi.WithOptions(options.Api.Offline(true))
@@ -42,14 +43,16 @@ func NewIpfsBstore(ctx context.Context, onlineMode bool) (*IpfsBstore, error) {
 		}
 	}
 
-	return &IpfsBstore{
+	bs := &IPFSBlockstore{
 		ctx:        ctx,
 		api:        api,
 		offlineAPI: offlineAPI,
-	}, nil
+	}
+
+	return Adapt(bs), nil
 }
 
-func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (*IpfsBstore, error) {
+func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error) {
 	httpApi, err := httpapi.NewApi(maddr)
 	if err != nil {
 		return nil, xerrors.Errorf("setting remote ipfs api: %w", err)
@@ -58,6 +61,7 @@ func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineM
 	if err != nil {
 		return nil, xerrors.Errorf("applying offline mode: %s", err)
 	}
+
 	offlineAPI := api
 	if onlineMode {
 		offlineAPI, err = httpApi.WithOptions(options.Api.Offline(true))
@@ -66,18 +70,20 @@ func NewRemoteIpfsBstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineM
 		}
 	}
 
-	return &IpfsBstore{
+	bs := &IPFSBlockstore{
 		ctx:        ctx,
 		api:        api,
 		offlineAPI: offlineAPI,
-	}, nil
+	}
+
+	return Adapt(bs), nil
 }
 
-func (i *IpfsBstore) DeleteBlock(cid cid.Cid) error {
+func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
 	return xerrors.Errorf("not supported")
 }
 
-func (i *IpfsBstore) Has(cid cid.Cid) (bool, error) {
+func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
 	_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
 	if err != nil {
 		// The underlying client is running in Offline mode.
@@ -93,7 +99,7 @@ func (i *IpfsBstore) Has(cid cid.Cid) (bool, error) {
 	return true, nil
 }
 
-func (i *IpfsBstore) Get(cid cid.Cid) (blocks.Block, error) {
+func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
 	rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
 	if err != nil {
 		return nil, xerrors.Errorf("getting ipfs block: %w", err)
@@ -107,7 +113,7 @@ func (i *IpfsBstore) Get(cid cid.Cid) (blocks.Block, error) {
 	return blocks.NewBlockWithCid(data, cid)
 }
 
-func (i *IpfsBstore) GetSize(cid cid.Cid) (int, error) {
+func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
 	st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
 	if err != nil {
 		return 0, xerrors.Errorf("getting ipfs block: %w", err)
@@ -116,7 +122,7 @@ func (i *IpfsBstore) GetSize(cid cid.Cid) (int, error) {
 	return st.Size(), nil
 }
 
-func (i *IpfsBstore) Put(block blocks.Block) error {
+func (i *IPFSBlockstore) Put(block blocks.Block) error {
 	mhd, err := multihash.Decode(block.Cid().Hash())
 	if err != nil {
 		return err
@@ -128,7 +134,7 @@ func (i *IpfsBstore) Put(block blocks.Block) error {
 	return err
 }
 
-func (i *IpfsBstore) PutMany(blocks []blocks.Block) error {
+func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
 	// TODO: could be done in parallel
 
 	for _, block := range blocks {
@@ -140,12 +146,10 @@ func (i *IpfsBstore) PutMany(blocks []blocks.Block) error {
 	return nil
 }
 
-func (i *IpfsBstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+func (i *IPFSBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	return nil, xerrors.Errorf("not supported")
 }
 
-func (i *IpfsBstore) HashOnRead(enabled bool) {
+func (i *IPFSBlockstore) HashOnRead(enabled bool) {
 	return // TODO: We could technically support this, but..
 }
-
-var _ blockstore.Blockstore = &IpfsBstore{}

blockstore/mem.go

@@ -7,20 +7,32 @@ import (
 	"github.com/ipfs/go-cid"
 )
 
-// MemStore is a terminal blockstore that keeps blocks in memory.
+// NewMemory returns a temporary memory-backed blockstore.
-type MemStore map[cid.Cid]blocks.Block
+func NewMemory() MemBlockstore {
+	return make(MemBlockstore)
+}
 
-func (m MemStore) DeleteBlock(k cid.Cid) error {
+// MemBlockstore is a terminal blockstore that keeps blocks in memory.
+type MemBlockstore map[cid.Cid]blocks.Block
+
+func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
 	delete(m, k)
 	return nil
 }
 
-func (m MemStore) Has(k cid.Cid) (bool, error) {
+func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
+	for _, k := range ks {
+		delete(m, k)
+	}
+	return nil
+}
+
+func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
 	_, ok := m[k]
 	return ok, nil
 }
 
-func (m MemStore) View(k cid.Cid, callback func([]byte) error) error {
+func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
 	b, ok := m[k]
 	if !ok {
 		return ErrNotFound
@@ -28,7 +40,7 @@ func (m MemStore) View(k cid.Cid, callback func([]byte) error) error {
 	return callback(b.RawData())
 }
 
-func (m MemStore) Get(k cid.Cid) (blocks.Block, error) {
+func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
 	b, ok := m[k]
 	if !ok {
 		return nil, ErrNotFound
@@ -37,7 +49,7 @@ func (m MemStore) Get(k cid.Cid) (blocks.Block, error) {
 }
 
 // GetSize returns the CIDs mapped BlockSize
-func (m MemStore) GetSize(k cid.Cid) (int, error) {
+func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
 	b, ok := m[k]
 	if !ok {
 		return 0, ErrNotFound
@@ -46,7 +58,7 @@ func (m MemStore) GetSize(k cid.Cid) (int, error) {
 }
 
 // Put puts a given block to the underlying datastore
-func (m MemStore) Put(b blocks.Block) error {
+func (m MemBlockstore) Put(b blocks.Block) error {
 	// Convert to a basic block for safety, but try to reuse the existing
 	// block if it's already a basic block.
 	k := b.Cid()
@@ -64,7 +76,7 @@ func (m MemStore) Put(b blocks.Block) error {
 
 // PutMany puts a slice of blocks at the same time using batching
 // capabilities of the underlying datastore whenever possible.
-func (m MemStore) PutMany(bs []blocks.Block) error {
+func (m MemBlockstore) PutMany(bs []blocks.Block) error {
 	for _, b := range bs {
 		_ = m.Put(b) // can't fail
 	}
@@ -74,7 +86,7 @@ func (m MemStore) PutMany(bs []blocks.Block) error {
 // AllKeysChan returns a channel from which
 // the CIDs in the Blockstore can be read. It should respect
 // the given context, closing the channel if it becomes Done.
-func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	ch := make(chan cid.Cid, len(m))
 	for k := range m {
 		ch <- k
@@ -85,6 +97,6 @@ func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 
 // HashOnRead specifies if every read block should be
 // rehashed to make sure it matches its CID.
-func (m MemStore) HashOnRead(enabled bool) {
+func (m MemBlockstore) HashOnRead(enabled bool) {
 	// no-op
 }

blockstore/metrics.go

@@ -0,0 +1,154 @@
+package blockstore
+
+import (
+	"time"
+
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+//
+// Currently unused, but kept in repo in case we introduce one of the candidate
+// cache implementations (Freecache, Ristretto), both of which report these
+// metrics.
+//
+
+// CacheMetricsEmitInterval is the interval at which metrics are emitted onto
+// OpenCensus.
+var CacheMetricsEmitInterval = 5 * time.Second
+
+var (
+	CacheName, _ = tag.NewKey("cache_name")
+)
+
+// CacheMeasures groups all metrics emitted by the blockstore caches.
+var CacheMeasures = struct {
+	HitRatio       *stats.Float64Measure
+	Hits           *stats.Int64Measure
+	Misses         *stats.Int64Measure
+	Entries        *stats.Int64Measure
+	QueriesServed  *stats.Int64Measure
+	Adds           *stats.Int64Measure
+	Updates        *stats.Int64Measure
+	Evictions      *stats.Int64Measure
+	CostAdded      *stats.Int64Measure
+	CostEvicted    *stats.Int64Measure
+	SetsDropped    *stats.Int64Measure
+	SetsRejected   *stats.Int64Measure
+	QueriesDropped *stats.Int64Measure
+}{
+	HitRatio:       stats.Float64("blockstore/cache/hit_ratio", "Hit ratio of blockstore cache", stats.UnitDimensionless),
+	Hits:           stats.Int64("blockstore/cache/hits", "Total number of hits at blockstore cache", stats.UnitDimensionless),
+	Misses:         stats.Int64("blockstore/cache/misses", "Total number of misses at blockstore cache", stats.UnitDimensionless),
+	Entries:        stats.Int64("blockstore/cache/entry_count", "Total number of entries currently in the blockstore cache", stats.UnitDimensionless),
+	QueriesServed:  stats.Int64("blockstore/cache/queries_served", "Total number of queries served by the blockstore cache", stats.UnitDimensionless),
+	Adds:           stats.Int64("blockstore/cache/adds", "Total number of adds to blockstore cache", stats.UnitDimensionless),
+	Updates:        stats.Int64("blockstore/cache/updates", "Total number of updates in blockstore cache", stats.UnitDimensionless),
+	Evictions:      stats.Int64("blockstore/cache/evictions", "Total number of evictions from blockstore cache", stats.UnitDimensionless),
+	CostAdded:      stats.Int64("blockstore/cache/cost_added", "Total cost (byte size) of entries added into blockstore cache", stats.UnitBytes),
+	CostEvicted:    stats.Int64("blockstore/cache/cost_evicted", "Total cost (byte size) of entries evicted by blockstore cache", stats.UnitBytes),
+	SetsDropped:    stats.Int64("blockstore/cache/sets_dropped", "Total number of sets dropped by blockstore cache", stats.UnitDimensionless),
+	SetsRejected:   stats.Int64("blockstore/cache/sets_rejected", "Total number of sets rejected by blockstore cache", stats.UnitDimensionless),
+	QueriesDropped: stats.Int64("blockstore/cache/queries_dropped", "Total number of queries dropped by blockstore cache", stats.UnitDimensionless),
+}
+
+// CacheViews groups all cache-related default views.
+var CacheViews = struct {
+	HitRatio       *view.View
+	Hits           *view.View
+	Misses         *view.View
+	Entries        *view.View
+	QueriesServed  *view.View
+	Adds           *view.View
+	Updates        *view.View
+	Evictions      *view.View
+	CostAdded      *view.View
+	CostEvicted    *view.View
+	SetsDropped    *view.View
+	SetsRejected   *view.View
+	QueriesDropped *view.View
+}{
+	HitRatio: &view.View{
+		Measure:     CacheMeasures.HitRatio,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Hits: &view.View{
+		Measure:     CacheMeasures.Hits,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Misses: &view.View{
+		Measure:     CacheMeasures.Misses,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Entries: &view.View{
+		Measure:     CacheMeasures.Entries,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	QueriesServed: &view.View{
+		Measure:     CacheMeasures.QueriesServed,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Adds: &view.View{
+		Measure:     CacheMeasures.Adds,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Updates: &view.View{
+		Measure:     CacheMeasures.Updates,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	Evictions: &view.View{
+		Measure:     CacheMeasures.Evictions,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	CostAdded: &view.View{
+		Measure:     CacheMeasures.CostAdded,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	CostEvicted: &view.View{
+		Measure:     CacheMeasures.CostEvicted,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	SetsDropped: &view.View{
+		Measure:     CacheMeasures.SetsDropped,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	SetsRejected: &view.View{
+		Measure:     CacheMeasures.SetsRejected,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+	QueriesDropped: &view.View{
+		Measure:     CacheMeasures.QueriesDropped,
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{CacheName},
+	},
+}
+
+// DefaultViews exports all default views for this package.
+var DefaultViews = []*view.View{
+	CacheViews.HitRatio,
+	CacheViews.Hits,
+	CacheViews.Misses,
+	CacheViews.Entries,
+	CacheViews.QueriesServed,
+	CacheViews.Adds,
+	CacheViews.Updates,
+	CacheViews.Evictions,
+	CacheViews.CostAdded,
+	CacheViews.CostEvicted,
+	CacheViews.SetsDropped,
+	CacheViews.SetsRejected,
+	CacheViews.QueriesDropped,
+}

blockstore/splitstore/markset.go

@@ -0,0 +1,38 @@
+package splitstore
+
+import (
+	"path/filepath"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+)
+
+// MarkSet is a utility to keep track of seen CID, and later query for them.
+//
+// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
+// * If a probabilistic result is acceptable, it can be backed by a bloom filter (default).
+type MarkSet interface {
+	Mark(cid.Cid) error
+	Has(cid.Cid) (bool, error)
+	Close() error
+}
+
+// markBytes is deliberately a non-nil empty byte slice for serialization.
+var markBytes = []byte{}
+
+type MarkSetEnv interface {
+	Create(name string, sizeHint int64) (MarkSet, error)
+	Close() error
+}
+
+func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
+	switch mtype {
+	case "", "bloom":
+		return NewBloomMarkSetEnv()
+	case "bolt":
+		return NewBoltMarkSetEnv(filepath.Join(path, "markset.bolt"))
+	default:
+		return nil, xerrors.Errorf("unknown mark set type %s", mtype)
+	}
+}

blockstore/splitstore/markset_bloom.go

@@ -0,0 +1,77 @@
+package splitstore
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+
+	"golang.org/x/xerrors"
+
+	bbloom "github.com/ipfs/bbloom"
+	cid "github.com/ipfs/go-cid"
+)
+
+const (
+	BloomFilterMinSize     = 10_000_000
+	BloomFilterProbability = 0.01
+)
+
+type BloomMarkSetEnv struct{}
+
+var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
+
+type BloomMarkSet struct {
+	salt []byte
+	bf   *bbloom.Bloom
+}
+
+var _ MarkSet = (*BloomMarkSet)(nil)
+
+func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error) {
+	return &BloomMarkSetEnv{}, nil
+}
+
+func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
+	size := int64(BloomFilterMinSize)
+	for size < sizeHint {
+		size += BloomFilterMinSize
+	}
+
+	salt := make([]byte, 4)
+	_, err := rand.Read(salt)
+	if err != nil {
+		return nil, xerrors.Errorf("error reading salt: %w", err)
+	}
+
+	bf, err := bbloom.New(float64(size), BloomFilterProbability)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating bloom filter: %w", err)
+	}
+
+	return &BloomMarkSet{salt: salt, bf: bf}, nil
+}
+
+func (e *BloomMarkSetEnv) Close() error {
+	return nil
+}
+
+func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
+	hash := cid.Hash()
+	key := make([]byte, len(s.salt)+len(hash))
+	n := copy(key, s.salt)
+	copy(key[n:], hash)
+	rehash := sha256.Sum256(key)
+	return rehash[:]
+}
+
+func (s *BloomMarkSet) Mark(cid cid.Cid) error {
+	s.bf.Add(s.saltedKey(cid))
+	return nil
+}
+
+func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
+	return s.bf.Has(s.saltedKey(cid)), nil
+}
+
+func (s *BloomMarkSet) Close() error {
+	return nil
+}

blockstore/splitstore/markset_bolt.go

@@ -0,0 +1,81 @@
+package splitstore
+
+import (
+	"time"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+	bolt "go.etcd.io/bbolt"
+)
+
+type BoltMarkSetEnv struct {
+	db *bolt.DB
+}
+
+var _ MarkSetEnv = (*BoltMarkSetEnv)(nil)
+
+type BoltMarkSet struct {
+	db       *bolt.DB
+	bucketId []byte
+}
+
+var _ MarkSet = (*BoltMarkSet)(nil)
+
+func NewBoltMarkSetEnv(path string) (*BoltMarkSetEnv, error) {
+	db, err := bolt.Open(path, 0644,
+		&bolt.Options{
+			Timeout: 1 * time.Second,
+			NoSync:  true,
+		})
+	if err != nil {
+		return nil, err
+	}
+
+	return &BoltMarkSetEnv{db: db}, nil
+}
+
+func (e *BoltMarkSetEnv) Create(name string, hint int64) (MarkSet, error) {
+	bucketId := []byte(name)
+	err := e.db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(bucketId)
+		if err != nil {
+			return xerrors.Errorf("error creating bolt db bucket %s: %w", name, err)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &BoltMarkSet{db: e.db, bucketId: bucketId}, nil
+}
+
+func (e *BoltMarkSetEnv) Close() error {
+	return e.db.Close()
+}
+
+func (s *BoltMarkSet) Mark(cid cid.Cid) error {
+	return s.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		return b.Put(cid.Hash(), markBytes)
+	})
+}
+
+func (s *BoltMarkSet) Has(cid cid.Cid) (result bool, err error) {
+	err = s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		v := b.Get(cid.Hash())
+		result = v != nil
+		return nil
+	})
+
+	return result, err
+}
+
+func (s *BoltMarkSet) Close() error {
+	return s.db.Update(func(tx *bolt.Tx) error {
+		return tx.DeleteBucket(s.bucketId)
+	})
+}

blockstore/splitstore/markset_test.go

@@ -0,0 +1,138 @@
+package splitstore
+
+import (
+	"io/ioutil"
+	"testing"
+
+	cid "github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestBoltMarkSet(t *testing.T) {
+	testMarkSet(t, "bolt")
+}
+
+func TestBloomMarkSet(t *testing.T) {
+	testMarkSet(t, "bloom")
+}
+
+func testMarkSet(t *testing.T, lsType string) {
+	t.Helper()
+
+	path, err := ioutil.TempDir("", "sweep-test.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	hotSet, err := env.Create("hot", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	coldSet, err := env.Create("cold", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	hotSet.Mark(k1)  //nolint
+	hotSet.Mark(k2)  //nolint
+	coldSet.Mark(k3) //nolint
+
+	mustHave(hotSet, k1)
+	mustHave(hotSet, k2)
+	mustNotHave(hotSet, k3)
+	mustNotHave(hotSet, k4)
+
+	mustNotHave(coldSet, k1)
+	mustNotHave(coldSet, k2)
+	mustHave(coldSet, k3)
+	mustNotHave(coldSet, k4)
+
+	// close them and reopen to redo the dance
+
+	err = hotSet.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = coldSet.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hotSet, err = env.Create("hot", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	coldSet, err = env.Create("cold", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hotSet.Mark(k3)  //nolint
+	hotSet.Mark(k4)  //nolint
+	coldSet.Mark(k1) //nolint
+
+	mustNotHave(hotSet, k1)
+	mustNotHave(hotSet, k2)
+	mustHave(hotSet, k3)
+	mustHave(hotSet, k4)
+
+	mustHave(coldSet, k1)
+	mustNotHave(coldSet, k2)
+	mustNotHave(coldSet, k3)
+	mustNotHave(coldSet, k4)
+
+	err = hotSet.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = coldSet.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

blockstore/splitstore/splitstore_test.go

@@ -0,0 +1,255 @@
+package splitstore
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+
+	cid "github.com/ipfs/go-cid"
+	datastore "github.com/ipfs/go-datastore"
+	dssync "github.com/ipfs/go-datastore/sync"
+	logging "github.com/ipfs/go-log/v2"
+)
+
+func init() {
+	CompactionThreshold = 5
+	CompactionCold = 1
+	CompactionBoundary = 2
+	logging.SetLogLevel("splitstore", "DEBUG")
+}
+
+func testSplitStore(t *testing.T, cfg *Config) {
+	chain := &mockChain{}
+	// genesis
+	genBlock := mock.MkBlock(nil, 0, 0)
+	genTs := mock.TipSet(genBlock)
+	chain.push(genTs)
+
+	// the myriads of stores
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := blockstore.NewMemorySync()
+	cold := blockstore.NewMemorySync()
+
+	// put the genesis block to cold store
+	blk, err := genBlock.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = cold.Put(blk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// open the splitstore
+	ss, err := Open("", ds, hot, cold, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	err = ss.Start(chain)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// make some tipsets, but not enough to cause compaction
+	mkBlock := func(curTs *types.TipSet, i int) *types.TipSet {
+		blk := mock.MkBlock(curTs, uint64(i), uint64(i))
+		sblk, err := blk.ToStorageBlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ss.Put(sblk)
+		if err != nil {
+			t.Fatal(err)
+		}
+		ts := mock.TipSet(blk)
+		chain.push(ts)
+
+		return ts
+	}
+
+	mkGarbageBlock := func(curTs *types.TipSet, i int) {
+		blk := mock.MkBlock(curTs, uint64(i), uint64(i))
+		sblk, err := blk.ToStorageBlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ss.Put(sblk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	waitForCompaction := func() {
+		for atomic.LoadInt32(&ss.compacting) == 1 {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+
+	curTs := genTs
+	for i := 1; i < 5; i++ {
+		curTs = mkBlock(curTs, i)
+		waitForCompaction()
+	}
+
+	mkGarbageBlock(genTs, 1)
+
+	// count objects in the cold and hot stores
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	countBlocks := func(bs blockstore.Blockstore) int {
+		count := 0
+		ch, err := bs.AllKeysChan(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for range ch {
+			count++
+		}
+		return count
+	}
+
+	coldCnt := countBlocks(cold)
+	hotCnt := countBlocks(hot)
+
+	if coldCnt != 1 {
+		t.Errorf("expected %d blocks, but got %d", 1, coldCnt)
+	}
+
+	if hotCnt != 5 {
+		t.Errorf("expected %d blocks, but got %d", 5, hotCnt)
+	}
+
+	// trigger a compaction
+	for i := 5; i < 10; i++ {
+		curTs = mkBlock(curTs, i)
+		waitForCompaction()
+	}
+
+	coldCnt = countBlocks(cold)
+	hotCnt = countBlocks(hot)
+
+	if !cfg.EnableFullCompaction {
+		if coldCnt != 5 {
+			t.Errorf("expected %d cold blocks, but got %d", 5, coldCnt)
+		}
+
+		if hotCnt != 5 {
+			t.Errorf("expected %d hot blocks, but got %d", 5, hotCnt)
+		}
+	}
+
+	if cfg.EnableFullCompaction && !cfg.EnableGC {
+		if coldCnt != 3 {
+			t.Errorf("expected %d cold blocks, but got %d", 3, coldCnt)
+		}
+
+		if hotCnt != 7 {
+			t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
+		}
+	}
+
+	if cfg.EnableFullCompaction && cfg.EnableGC {
+		if coldCnt != 2 {
+			t.Errorf("expected %d cold blocks, but got %d", 2, coldCnt)
+		}
+
+		if hotCnt != 7 {
+			t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
+		}
+	}
+}
+
+func TestSplitStoreSimpleCompaction(t *testing.T) {
+	testSplitStore(t, &Config{TrackingStoreType: "mem"})
+}
+
+func TestSplitStoreFullCompactionWithoutGC(t *testing.T) {
+	testSplitStore(t, &Config{
+		TrackingStoreType:    "mem",
+		EnableFullCompaction: true,
+	})
+}
+
+func TestSplitStoreFullCompactionWithGC(t *testing.T) {
+	testSplitStore(t, &Config{
+		TrackingStoreType:    "mem",
+		EnableFullCompaction: true,
+		EnableGC:             true,
+	})
+}
+
+type mockChain struct {
+	sync.Mutex
+	tipsets  []*types.TipSet
+	listener func(revert []*types.TipSet, apply []*types.TipSet) error
+}
+
+func (c *mockChain) push(ts *types.TipSet) {
+	c.Lock()
+	c.tipsets = append(c.tipsets, ts)
+	c.Unlock()
+
+	if c.listener != nil {
+		err := c.listener(nil, []*types.TipSet{ts})
+		if err != nil {
+			log.Errorf("mockchain: error dispatching listener: %s", err)
+		}
+	}
+}
+
+func (c *mockChain) GetTipsetByHeight(_ context.Context, epoch abi.ChainEpoch, _ *types.TipSet, _ bool) (*types.TipSet, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	iEpoch := int(epoch)
+	if iEpoch > len(c.tipsets) {
+		return nil, fmt.Errorf("bad epoch %d", epoch)
+	}
+
+	return c.tipsets[iEpoch-1], nil
+}
+
+func (c *mockChain) GetHeaviestTipSet() *types.TipSet {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.tipsets[len(c.tipsets)-1]
+}
+
+func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error) {
+	c.listener = change
+}
+
+func (c *mockChain) WalkSnapshot(_ context.Context, ts *types.TipSet, epochs abi.ChainEpoch, _ bool, _ bool, f func(cid.Cid) error) error {
+	c.Lock()
+	defer c.Unlock()
+
+	start := int(ts.Height()) - 1
+	end := start - int(epochs)
+	if end < 0 {
+		end = -1
+	}
+	for i := start; i > end; i-- {
+		ts := c.tipsets[i]
+		for _, cid := range ts.Cids() {
+			err := f(cid)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

blockstore/splitstore/tracking.go

@@ -0,0 +1,109 @@
+package splitstore
+
+import (
+	"path/filepath"
+	"sync"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	cid "github.com/ipfs/go-cid"
+)
+
+// TrackingStore is a persistent store that tracks blocks that are added
+// to the hotstore, tracking the epoch at which they are written.
+type TrackingStore interface {
+	Put(cid.Cid, abi.ChainEpoch) error
+	PutBatch([]cid.Cid, abi.ChainEpoch) error
+	Get(cid.Cid) (abi.ChainEpoch, error)
+	Delete(cid.Cid) error
+	DeleteBatch([]cid.Cid) error
+	ForEach(func(cid.Cid, abi.ChainEpoch) error) error
+	Sync() error
+	Close() error
+}
+
+// OpenTrackingStore opens a tracking store of the specified type in the
+// specified path.
+func OpenTrackingStore(path string, ttype string) (TrackingStore, error) {
+	switch ttype {
+	case "", "bolt":
+		return OpenBoltTrackingStore(filepath.Join(path, "tracker.bolt"))
+	case "mem":
+		return NewMemTrackingStore(), nil
+	default:
+		return nil, xerrors.Errorf("unknown tracking store type %s", ttype)
+	}
+}
+
+// NewMemTrackingStore creates an in-memory tracking store.
+// This is only useful for test or situations where you don't want to open the
+// real tracking store (eg concurrent read only access on a node's datastore)
+func NewMemTrackingStore() *MemTrackingStore {
+	return &MemTrackingStore{tab: make(map[cid.Cid]abi.ChainEpoch)}
+}
+
+// MemTrackingStore is a simple in-memory tracking store
+type MemTrackingStore struct {
+	sync.Mutex
+	tab map[cid.Cid]abi.ChainEpoch
+}
+
+var _ TrackingStore = (*MemTrackingStore)(nil)
+
+func (s *MemTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
+	s.Lock()
+	defer s.Unlock()
+	s.tab[cid] = epoch
+	return nil
+}
+
+func (s *MemTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
+	s.Lock()
+	defer s.Unlock()
+	for _, cid := range cids {
+		s.tab[cid] = epoch
+	}
+	return nil
+}
+
+func (s *MemTrackingStore) Get(cid cid.Cid) (abi.ChainEpoch, error) {
+	s.Lock()
+	defer s.Unlock()
+	epoch, ok := s.tab[cid]
+	if ok {
+		return epoch, nil
+	}
+	return 0, xerrors.Errorf("missing tracking epoch for %s", cid)
+}
+
+func (s *MemTrackingStore) Delete(cid cid.Cid) error {
+	s.Lock()
+	defer s.Unlock()
+	delete(s.tab, cid)
+	return nil
+}
+
+func (s *MemTrackingStore) DeleteBatch(cids []cid.Cid) error {
+	s.Lock()
+	defer s.Unlock()
+	for _, cid := range cids {
+		delete(s.tab, cid)
+	}
+	return nil
+}
+
+func (s *MemTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
+	s.Lock()
+	defer s.Unlock()
+	for cid, epoch := range s.tab {
+		err := f(cid, epoch)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *MemTrackingStore) Sync() error  { return nil }
+func (s *MemTrackingStore) Close() error { return nil }

blockstore/splitstore/tracking_bolt.go

@@ -0,0 +1,120 @@
+package splitstore
+
+import (
+	"time"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+	bolt "go.etcd.io/bbolt"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+type BoltTrackingStore struct {
+	db       *bolt.DB
+	bucketId []byte
+}
+
+var _ TrackingStore = (*BoltTrackingStore)(nil)
+
+func OpenBoltTrackingStore(path string) (*BoltTrackingStore, error) {
+	opts := &bolt.Options{
+		Timeout: 1 * time.Second,
+		NoSync:  true,
+	}
+	db, err := bolt.Open(path, 0644, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	bucketId := []byte("tracker")
+	err = db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(bucketId)
+		if err != nil {
+			return xerrors.Errorf("error creating bolt db bucket %s: %w", string(bucketId), err)
+		}
+		return nil
+	})
+
+	if err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+
+	return &BoltTrackingStore{db: db, bucketId: bucketId}, nil
+}
+
+func (s *BoltTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
+	val := epochToBytes(epoch)
+	return s.db.Batch(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		return b.Put(cid.Hash(), val)
+	})
+}
+
+func (s *BoltTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
+	val := epochToBytes(epoch)
+	return s.db.Batch(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		for _, cid := range cids {
+			err := b.Put(cid.Hash(), val)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func (s *BoltTrackingStore) Get(cid cid.Cid) (epoch abi.ChainEpoch, err error) {
+	err = s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		val := b.Get(cid.Hash())
+		if val == nil {
+			return xerrors.Errorf("missing tracking epoch for %s", cid)
+		}
+		epoch = bytesToEpoch(val)
+		return nil
+	})
+	return epoch, err
+}
+
+func (s *BoltTrackingStore) Delete(cid cid.Cid) error {
+	return s.db.Batch(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		return b.Delete(cid.Hash())
+	})
+}
+
+func (s *BoltTrackingStore) DeleteBatch(cids []cid.Cid) error {
+	return s.db.Batch(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		for _, cid := range cids {
+			err := b.Delete(cid.Hash())
+			if err != nil {
+				return xerrors.Errorf("error deleting %s", cid)
+			}
+		}
+		return nil
+	})
+}
+
+func (s *BoltTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
+	return s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(s.bucketId)
+		return b.ForEach(func(k, v []byte) error {
+			cid := cid.NewCidV1(cid.Raw, k)
+			epoch := bytesToEpoch(v)
+			return f(cid, epoch)
+		})
+	})
+}
+
+func (s *BoltTrackingStore) Sync() error {
+	return s.db.Sync()
+}
+
+func (s *BoltTrackingStore) Close() error {
+	return s.db.Close()
+}

blockstore/splitstore/tracking_test.go

@@ -0,0 +1,130 @@
+package splitstore
+
+import (
+	"io/ioutil"
+	"testing"
+
+	cid "github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+func TestBoltTrackingStore(t *testing.T) {
+	testTrackingStore(t, "bolt")
+}
+
+func testTrackingStore(t *testing.T, tsType string) {
+	t.Helper()
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s TrackingStore, cid cid.Cid, epoch abi.ChainEpoch) {
+		val, err := s.Get(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if val != epoch {
+			t.Fatal("epoch mismatch")
+		}
+	}
+
+	mustNotHave := func(s TrackingStore, cid cid.Cid) {
+		_, err := s.Get(cid)
+		if err == nil {
+			t.Fatal("expected error")
+		}
+	}
+
+	path, err := ioutil.TempDir("", "snoop-test.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := OpenTrackingStore(path, tsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	s.Put(k1, 1) //nolint
+	s.Put(k2, 2) //nolint
+	s.Put(k3, 3) //nolint
+	s.Put(k4, 4) //nolint
+
+	mustHave(s, k1, 1)
+	mustHave(s, k2, 2)
+	mustHave(s, k3, 3)
+	mustHave(s, k4, 4)
+
+	s.Delete(k1) // nolint
+	s.Delete(k2) // nolint
+
+	mustNotHave(s, k1)
+	mustNotHave(s, k2)
+	mustHave(s, k3, 3)
+	mustHave(s, k4, 4)
+
+	s.PutBatch([]cid.Cid{k1}, 1) //nolint
+	s.PutBatch([]cid.Cid{k2}, 2) //nolint
+
+	mustHave(s, k1, 1)
+	mustHave(s, k2, 2)
+	mustHave(s, k3, 3)
+	mustHave(s, k4, 4)
+
+	allKeys := map[string]struct{}{
+		k1.String(): {},
+		k2.String(): {},
+		k3.String(): {},
+		k4.String(): {},
+	}
+
+	err = s.ForEach(func(k cid.Cid, _ abi.ChainEpoch) error {
+		_, ok := allKeys[k.String()]
+		if !ok {
+			t.Fatal("unexpected key")
+		}
+
+		delete(allKeys, k.String())
+		return nil
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(allKeys) != 0 {
+		t.Fatal("not all keys were returned")
+	}
+
+	// no close and reopen and ensure the keys still exist
+	err = s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err = OpenTrackingStore(path, tsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(s, k1, 1)
+	mustHave(s, k2, 2)
+	mustHave(s, k3, 3)
+	mustHave(s, k4, 4)
+
+	s.Close() //nolint:errcheck
+}

blockstore/sync.go

@@ -0,0 +1,81 @@
+package blockstore
+
+import (
+	"context"
+	"sync"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+)
+
+// NewMemorySync returns a thread-safe in-memory blockstore.
+func NewMemorySync() *SyncBlockstore {
+	return &SyncBlockstore{bs: make(MemBlockstore)}
+}
+
+// SyncBlockstore is a terminal blockstore that is a synchronized version
+// of MemBlockstore.
+type SyncBlockstore struct {
+	mu sync.RWMutex
+	bs MemBlockstore // specifically use a memStore to save indirection overhead.
+}
+
+func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.bs.DeleteBlock(k)
+}
+
+func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.bs.DeleteMany(ks)
+}
+
+func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.bs.Has(k)
+}
+
+func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	return m.bs.View(k, callback)
+}
+
+func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.bs.Get(k)
+}
+
+func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.bs.GetSize(k)
+}
+
+func (m *SyncBlockstore) Put(b blocks.Block) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.bs.Put(b)
+}
+
+func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.bs.PutMany(bs)
+}
+
+func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	// this blockstore implementation doesn't do any async work.
+	return m.bs.AllKeysChan(ctx)
+}
+
+func (m *SyncBlockstore) HashOnRead(enabled bool) {
+	// noop
+}

blockstore/timed.go

@@ -1,4 +1,4 @@
-package timedbs
+package blockstore
 
 import (
 	"context"
@@ -10,37 +10,37 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/raulk/clock"
 	"go.uber.org/multierr"
-
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 )
 
-// TimedCacheBS is a blockstore that keeps blocks for at least the specified
+// TimedCacheBlockstore is a blockstore that keeps blocks for at least the
-// caching interval before discarding them. Garbage collection must be started
+// specified caching interval before discarding them. Garbage collection must
-// and stopped by calling Start/Stop.
+// be started and stopped by calling Start/Stop.
 //
 // Under the covers, it's implemented with an active and an inactive blockstore
 // that are rotated every cache time interval. This means all blocks will be
 // stored at most 2x the cache interval.
-type TimedCacheBS struct {
+//
+// Create a new instance by calling the NewTimedCacheBlockstore constructor.
+type TimedCacheBlockstore struct {
 	mu               sync.RWMutex
-	active, inactive blockstore.MemStore
+	active, inactive MemBlockstore
 	clock            clock.Clock
 	interval         time.Duration
 	closeCh          chan struct{}
 	doneRotatingCh   chan struct{}
 }
 
-func NewTimedCacheBS(cacheTime time.Duration) *TimedCacheBS {
+func NewTimedCacheBlockstore(interval time.Duration) *TimedCacheBlockstore {
-	return &TimedCacheBS{
+	b := &TimedCacheBlockstore{
-		active:   blockstore.NewTemporary(),
+		active:   NewMemory(),
-		inactive: blockstore.NewTemporary(),
+		inactive: NewMemory(),
-		interval: cacheTime,
+		interval: interval,
-		clock:    build.Clock,
+		clock:    clock.New(),
 	}
+	return b
 }
 
-func (t *TimedCacheBS) Start(ctx context.Context) error {
+func (t *TimedCacheBlockstore) Start(_ context.Context) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	if t.closeCh != nil {
@@ -65,11 +65,11 @@ func (t *TimedCacheBS) Start(ctx context.Context) error {
 	return nil
 }
 
-func (t *TimedCacheBS) Stop(ctx context.Context) error {
+func (t *TimedCacheBlockstore) Stop(_ context.Context) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	if t.closeCh == nil {
-		return fmt.Errorf("not started started")
+		return fmt.Errorf("not started")
 	}
 	select {
 	case <-t.closeCh:
@@ -80,15 +80,15 @@ func (t *TimedCacheBS) Stop(ctx context.Context) error {
 	return nil
 }
 
-func (t *TimedCacheBS) rotate() {
+func (t *TimedCacheBlockstore) rotate() {
-	newBs := blockstore.NewTemporary()
+	newBs := NewMemory()
 
 	t.mu.Lock()
 	t.inactive, t.active = t.active, newBs
 	t.mu.Unlock()
 }
 
-func (t *TimedCacheBS) Put(b blocks.Block) error {
+func (t *TimedCacheBlockstore) Put(b blocks.Block) error {
 	// Don't check the inactive set here. We want to keep this block for at
 	// least one interval.
 	t.mu.Lock()
@@ -96,33 +96,43 @@ func (t *TimedCacheBS) Put(b blocks.Block) error {
 	return t.active.Put(b)
 }
 
-func (t *TimedCacheBS) PutMany(bs []blocks.Block) error {
+func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	return t.active.PutMany(bs)
 }
 
-func (t *TimedCacheBS) Get(k cid.Cid) (blocks.Block, error) {
+func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+	err := t.active.View(k, callback)
+	if err == ErrNotFound {
+		err = t.inactive.View(k, callback)
+	}
+	return err
+}
+
+func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	b, err := t.active.Get(k)
-	if err == blockstore.ErrNotFound {
+	if err == ErrNotFound {
 		b, err = t.inactive.Get(k)
 	}
 	return b, err
 }
 
-func (t *TimedCacheBS) GetSize(k cid.Cid) (int, error) {
+func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	size, err := t.active.GetSize(k)
-	if err == blockstore.ErrNotFound {
+	if err == ErrNotFound {
 		size, err = t.inactive.GetSize(k)
 	}
 	return size, err
 }
 
-func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) {
+func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	if has, err := t.active.Has(k); err != nil {
@@ -133,17 +143,23 @@ func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) {
 	return t.inactive.Has(k)
 }
 
-func (t *TimedCacheBS) HashOnRead(_ bool) {
+func (t *TimedCacheBlockstore) HashOnRead(_ bool) {
 	// no-op
 }
 
-func (t *TimedCacheBS) DeleteBlock(k cid.Cid) error {
+func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k))
 }
 
-func (t *TimedCacheBS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	return multierr.Combine(t.active.DeleteMany(ks), t.inactive.DeleteMany(ks))
+}
+
+func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 

blockstore/timed_test.go

@@ -1,4 +1,4 @@
-package timedbs
+package blockstore
 
 import (
 	"context"
@@ -12,8 +12,8 @@ import (
 	"github.com/ipfs/go-cid"
 )
 
-func TestTimedBSSimple(t *testing.T) {
+func TestTimedCacheBlockstoreSimple(t *testing.T) {
-	tc := NewTimedCacheBS(10 * time.Millisecond)
+	tc := NewTimedCacheBlockstore(10 * time.Millisecond)
 	mClock := clock.NewMock()
 	mClock.Set(time.Now())
 	tc.clock = mClock

blockstore/union.go

@@ -0,0 +1,119 @@
+package blockstore
+
+import (
+	"context"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+)
+
+type unionBlockstore []Blockstore
+
+// Union returns an unioned blockstore.
+//
+// * Reads return from the first blockstore that has the value, querying in the
+//   supplied order.
+// * Writes (puts and deltes) are broadcast to all stores.
+//
+func Union(stores ...Blockstore) Blockstore {
+	return unionBlockstore(stores)
+}
+
+func (m unionBlockstore) Has(cid cid.Cid) (has bool, err error) {
+	for _, bs := range m {
+		if has, err = bs.Has(cid); has || err != nil {
+			break
+		}
+	}
+	return has, err
+}
+
+func (m unionBlockstore) Get(cid cid.Cid) (blk blocks.Block, err error) {
+	for _, bs := range m {
+		if blk, err = bs.Get(cid); err == nil || err != ErrNotFound {
+			break
+		}
+	}
+	return blk, err
+}
+
+func (m unionBlockstore) View(cid cid.Cid, callback func([]byte) error) (err error) {
+	for _, bs := range m {
+		if err = bs.View(cid, callback); err == nil || err != ErrNotFound {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) GetSize(cid cid.Cid) (size int, err error) {
+	for _, bs := range m {
+		if size, err = bs.GetSize(cid); err == nil || err != ErrNotFound {
+			break
+		}
+	}
+	return size, err
+}
+
+func (m unionBlockstore) Put(block blocks.Block) (err error) {
+	for _, bs := range m {
+		if err = bs.Put(block); err != nil {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) PutMany(blks []blocks.Block) (err error) {
+	for _, bs := range m {
+		if err = bs.PutMany(blks); err != nil {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) DeleteBlock(cid cid.Cid) (err error) {
+	for _, bs := range m {
+		if err = bs.DeleteBlock(cid); err != nil {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) DeleteMany(cids []cid.Cid) (err error) {
+	for _, bs := range m {
+		if err = bs.DeleteMany(cids); err != nil {
+			break
+		}
+	}
+	return err
+}
+
+func (m unionBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+	// this does not deduplicate; this interface needs to be revisited.
+	outCh := make(chan cid.Cid)
+
+	go func() {
+		defer close(outCh)
+
+		for _, bs := range m {
+			ch, err := bs.AllKeysChan(ctx)
+			if err != nil {
+				return
+			}
+			for cid := range ch {
+				outCh <- cid
+			}
+		}
+	}()
+
+	return outCh, nil
+}
+
+func (m unionBlockstore) HashOnRead(enabled bool) {
+	for _, bs := range m {
+		bs.HashOnRead(enabled)
+	}
+}

blockstore/union_test.go

@@ -0,0 +1,102 @@
+package blockstore
+
+import (
+	"context"
+	"testing"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	b0 = blocks.NewBlock([]byte("abc"))
+	b1 = blocks.NewBlock([]byte("foo"))
+	b2 = blocks.NewBlock([]byte("bar"))
+)
+
+func TestUnionBlockstore_Get(t *testing.T) {
+	m1 := NewMemory()
+	m2 := NewMemory()
+
+	_ = m1.Put(b1)
+	_ = m2.Put(b2)
+
+	u := Union(m1, m2)
+
+	v1, err := u.Get(b1.Cid())
+	require.NoError(t, err)
+	require.Equal(t, b1.RawData(), v1.RawData())
+
+	v2, err := u.Get(b2.Cid())
+	require.NoError(t, err)
+	require.Equal(t, b2.RawData(), v2.RawData())
+}
+
+func TestUnionBlockstore_Put_PutMany_Delete_AllKeysChan(t *testing.T) {
+	m1 := NewMemory()
+	m2 := NewMemory()
+
+	u := Union(m1, m2)
+
+	err := u.Put(b0)
+	require.NoError(t, err)
+
+	var has bool
+
+	// write was broadcasted to all stores.
+	has, _ = m1.Has(b0.Cid())
+	require.True(t, has)
+
+	has, _ = m2.Has(b0.Cid())
+	require.True(t, has)
+
+	has, _ = u.Has(b0.Cid())
+	require.True(t, has)
+
+	// put many.
+	err = u.PutMany([]blocks.Block{b1, b2})
+	require.NoError(t, err)
+
+	// write was broadcasted to all stores.
+	has, _ = m1.Has(b1.Cid())
+	require.True(t, has)
+
+	has, _ = m1.Has(b2.Cid())
+	require.True(t, has)
+
+	has, _ = m2.Has(b1.Cid())
+	require.True(t, has)
+
+	has, _ = m2.Has(b2.Cid())
+	require.True(t, has)
+
+	// also in the union store.
+	has, _ = u.Has(b1.Cid())
+	require.True(t, has)
+
+	has, _ = u.Has(b2.Cid())
+	require.True(t, has)
+
+	// deleted from all stores.
+	err = u.DeleteBlock(b1.Cid())
+	require.NoError(t, err)
+
+	has, _ = u.Has(b1.Cid())
+	require.False(t, has)
+
+	has, _ = m1.Has(b1.Cid())
+	require.False(t, has)
+
+	has, _ = m2.Has(b1.Cid())
+	require.False(t, has)
+
+	// check that AllKeysChan returns b0 and b2, twice (once per backing store)
+	ch, err := u.AllKeysChan(context.Background())
+	require.NoError(t, err)
+
+	var i int
+	for range ch {
+		i++
+	}
+	require.Equal(t, 4, i)
+}

build/bootstrap/butterflynet.pi

@@ -0,0 +1,2 @@
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWQafkXgEWDgcVhZvF6KMhiC8ktdxjvmdQN8RarRXe9jCc
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWE7UmZ4DLk9WBdEJUSwuSCPiSqjoCv3wPeoe8Tq3yMa77

build/bootstrap/calibnet.pi

@@ -1,4 +1,4 @@
-/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWK1QYsm6iqyhgH7vqsbeoNoKHbT368h1JLHS1qYN36oyc
+/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWRLZAseMo9h7fRD6ojn6YYDXHsBSavX5YmjBZ9ngtAEec
-/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWKDyJZoPsNak1iYNN1GGmvGnvhyVbWBL6iusYfP3RpgYs
+/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWJFtDXgZEQMEkjJPSrbfdvh2xfjVKrXeNFG1t8ioJXAzv
-/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWJRSTnzABB6MYYEBbSTT52phQntVD1PpRTMh1xt9mh6yH
+/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWP1uB9Lo7yCA3S17TD4Y5wStP5Nk7Vqh53m8GsFjkyujD
-/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWQLi3kY6HnMYLUtwCe26zWMdNhniFgHVNn1DioQc7NiWv
+/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWLrPM4WPK1YRGPCUwndWcDX8GCYgms3DiuofUmxwvhMCn

build/params_butterfly.go

@@ -0,0 +1,52 @@
+// +build butterflynet
+
+package build
+
+import (
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
+)
+
+var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
+	0: DrandMainnet,
+}
+
+const BootstrappersFile = "butterflynet.pi"
+const GenesisFile = "butterflynet.car"
+
+const UpgradeBreezeHeight = -1
+const BreezeGasTampingDuration = 120
+const UpgradeSmokeHeight = -2
+const UpgradeIgnitionHeight = -3
+const UpgradeRefuelHeight = -4
+
+var UpgradeActorsV2Height = abi.ChainEpoch(30)
+
+const UpgradeTapeHeight = 60
+const UpgradeLiftoffHeight = -5
+const UpgradeKumquatHeight = 90
+const UpgradeCalicoHeight = 120
+const UpgradePersianHeight = 150
+const UpgradeClausHeight = 180
+const UpgradeOrangeHeight = 210
+const UpgradeActorsV3Height = 240
+
+func init() {
+	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2 << 30))
+	policy.SetSupportedProofTypes(
+		abi.RegisteredSealProof_StackedDrg512MiBV1,
+	)
+
+	SetAddressNetwork(address.Testnet)
+
+	Devnet = true
+}
+
+const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
+
+const PropagationDelaySecs = uint64(6)
+
+// BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
+const BootstrapPeerThreshold = 2

build/params_calibnet.go

@@ -32,22 +32,18 @@ const UpgradeLiftoffHeight = -5
 
 const UpgradeKumquatHeight = 90
 
-const UpgradeCalicoHeight = 92000
+const UpgradeCalicoHeight = 100
-const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 60)
+const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 1)
 
-// 2020-12-17T19:00:00Z
+const UpgradeClausHeight = 250
-const UpgradeClausHeight = 161386
 
-// 2021-01-17T19:00:00Z
+const UpgradeOrangeHeight = 300
-const UpgradeOrangeHeight = 250666
 
-// 2021-01-28T21:00:00Z
+const UpgradeActorsV3Height = 600
-const UpgradeActorsV3Height = 282586
 
 func init() {
-	policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 30))
+	policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30))
 	policy.SetSupportedProofTypes(
-		abi.RegisteredSealProof_StackedDrg512MiBV1,
 		abi.RegisteredSealProof_StackedDrg32GiBV1,
 		abi.RegisteredSealProof_StackedDrg64GiBV1,
 	)

build/params_mainnet.go

@@ -3,10 +3,12 @@
 // +build !testground
 // +build !calibnet
 // +build !nerpanet
+// +build !butterflynet
 
 package build
 
 import (
+	"math"
 	"os"
 
 	"github.com/filecoin-project/go-address"
@@ -51,8 +53,8 @@ const UpgradeOrangeHeight = 336458
 // 2020-12-22T02:00:00Z
 const UpgradeClausHeight = 343200
 
-// TODO
+// 2021-03-04T00:00:30Z
-const UpgradeActorsV3Height = 999999999
+var UpgradeActorsV3Height = abi.ChainEpoch(550321)
 
 func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 40))
@@ -61,6 +63,10 @@ func init() {
 		SetAddressNetwork(address.Mainnet)
 	}
 
+	if os.Getenv("LOTUS_DISABLE_V3_ACTOR_MIGRATION") == "1" {
+		UpgradeActorsV3Height = math.MaxInt64
+	}
+
 	Devnet = false
 
 	BuildType = BuildMainnet

build/tools.go

@@ -3,5 +3,7 @@
 package build
 
 import (
+	_ "github.com/GeertJohan/go.rice/rice"
+	_ "github.com/golang/mock/mockgen"
 	_ "github.com/whyrusleeping/bencher"
 )

build/version.go

@@ -1,11 +1,5 @@
 package build
 
-import (
-	"fmt"
-
-	"golang.org/x/xerrors"
-)
-
 var CurrentCommit string
 var BuildType int
 
@@ -35,72 +29,8 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "1.4.2"
+const BuildVersion = "1.5.2"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit
 }
-
-type Version uint32
-
-func newVer(major, minor, patch uint8) Version {
-	return Version(uint32(major)<<16 | uint32(minor)<<8 | uint32(patch))
-}
-
-// Ints returns (major, minor, patch) versions
-func (ve Version) Ints() (uint32, uint32, uint32) {
-	v := uint32(ve)
-	return (v & majorOnlyMask) >> 16, (v & minorOnlyMask) >> 8, v & patchOnlyMask
-}
-
-func (ve Version) String() string {
-	vmj, vmi, vp := ve.Ints()
-	return fmt.Sprintf("%d.%d.%d", vmj, vmi, vp)
-}
-
-func (ve Version) EqMajorMinor(v2 Version) bool {
-	return ve&minorMask == v2&minorMask
-}
-
-type NodeType int
-
-const (
-	NodeUnknown NodeType = iota
-
-	NodeFull
-	NodeMiner
-	NodeWorker
-)
-
-var RunningNodeType NodeType
-
-func VersionForType(nodeType NodeType) (Version, error) {
-	switch nodeType {
-	case NodeFull:
-		return FullAPIVersion, nil
-	case NodeMiner:
-		return MinerAPIVersion, nil
-	case NodeWorker:
-		return WorkerAPIVersion, nil
-	default:
-		return Version(0), xerrors.Errorf("unknown node type %d", nodeType)
-	}
-}
-
-// semver versions of the rpc api exposed
-var (
-	FullAPIVersion   = newVer(1, 1, 0)
-	MinerAPIVersion  = newVer(1, 0, 1)
-	WorkerAPIVersion = newVer(1, 0, 0)
-)
-
-//nolint:varcheck,deadcode
-const (
-	majorMask = 0xff0000
-	minorMask = 0xffff00
-	patchMask = 0xffffff
-
-	majorOnlyMask = 0xff0000
-	minorOnlyMask = 0x00ff00
-	patchOnlyMask = 0x0000ff
-)

chain/actors/adt/diff_adt_test.go

@@ -16,7 +16,7 @@ import (
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 	adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
 
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
+	bstore "github.com/filecoin-project/lotus/blockstore"
 )
 
 func TestDiffAdtArray(t *testing.T) {
@@ -295,7 +295,7 @@ func (t *TestDiffArray) Remove(key uint64, val *typegen.Deferred) error {
 
 func newContextStore() Store {
 	ctx := context.Background()
-	bs := bstore.NewTemporarySync()
+	bs := bstore.NewMemorySync()
 	store := cbornode.NewCborStore(bs)
 	return WrapStore(ctx, store)
 }

chain/events/state/mock/api.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 )

chain/events/state/predicates.go

@@ -3,6 +3,7 @@ package state
 import (
 	"context"
 
+	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 
 	"github.com/filecoin-project/go-address"
@@ -10,7 +11,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
@@ -23,7 +24,7 @@ type UserData interface{}
 
 // ChainAPI abstracts out calls made by this class to external APIs
 type ChainAPI interface {
-	apibstore.ChainIO
+	api.ChainIO
 	StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error)
 }
 
@@ -36,7 +37,7 @@ type StatePredicates struct {
 func NewStatePredicates(api ChainAPI) *StatePredicates {
 	return &StatePredicates{
 		api: api,
-		cst: cbor.NewCborStore(apibstore.NewAPIBlockstore(api)),
+		cst: cbor.NewCborStore(blockstore.NewAPIBlockstore(api)),
 	}
 }
 

chain/events/state/predicates_test.go

@@ -23,9 +23,9 @@ import (
 	adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
 	tutils "github.com/filecoin-project/specs-actors/v2/support/testing"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 var dummyCid cid.Cid
@@ -36,7 +36,7 @@ func init() {
 
 func TestMarketPredicates(t *testing.T) {
 	ctx := context.Background()
-	bs := bstore.NewTemporarySync()
+	bs := bstore.NewMemorySync()
 	store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
 
 	oldDeal1 := &market2.DealState{
@@ -334,7 +334,7 @@ func TestMarketPredicates(t *testing.T) {
 
 func TestMinerSectorChange(t *testing.T) {
 	ctx := context.Background()
-	bs := bstore.NewTemporarySync()
+	bs := bstore.NewMemorySync()
 	store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs))
 
 	nextID := uint64(0)

chain/exchange/cbor_gen.go

@@ -5,6 +5,7 @@ package exchange
 import (
 	"fmt"
 	"io"
+	"sort"
 
 	types "github.com/filecoin-project/lotus/chain/types"
 	cid "github.com/ipfs/go-cid"
@@ -13,6 +14,8 @@ import (
 )
 
 var _ = xerrors.Errorf
+var _ = cid.Undef
+var _ = sort.Sort
 
 var lengthBufRequest = []byte{131}
 

chain/gen/gen.go

@@ -27,6 +27,7 @@ import (
 	proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/chain/beacon"
@@ -40,7 +41,6 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/genesis"
 	"github.com/filecoin-project/lotus/journal"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 	"github.com/filecoin-project/lotus/node/repo"
 )
@@ -125,7 +125,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 		return nil, xerrors.Errorf("failed to get metadata datastore: %w", err)
 	}
 
-	bs, err := lr.Blockstore(context.TODO(), repo.BlockstoreChain)
+	bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore)
 	if err != nil {
 		return nil, err
 	}

chain/gen/genesis/f00_system.go

@@ -8,8 +8,8 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupSystemActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/f01_init.go

@@ -16,9 +16,9 @@ import (
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/genesis"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesis.Actor, rootVerifier genesis.Actor, remainder genesis.Actor) (int64, *types.Actor, map[address.Address]address.Address, error) {

chain/gen/genesis/f02_reward.go

@@ -9,9 +9,9 @@ import (
 	reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupRewardActor(bs bstore.Blockstore, qaPower big.Int) (*types.Actor, error) {

chain/gen/genesis/f03_cron.go

@@ -7,8 +7,8 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/builtin/cron"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupCronActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/f04_power.go

@@ -9,8 +9,8 @@ import (
 	power0 "github.com/filecoin-project/specs-actors/actors/builtin/power"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/f05_market.go

@@ -8,8 +8,8 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) {

chain/gen/genesis/f06_vreg.go

@@ -10,8 +10,8 @@ import (
 	verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 var RootVerifierID address.Address

chain/gen/genesis/genesis.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
+
 	"github.com/filecoin-project/lotus/journal"
 
 	"github.com/ipfs/go-cid"
@@ -26,13 +27,13 @@ import (
 	verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 	adt0 "github.com/filecoin-project/specs-actors/actors/util/adt"
 
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/genesis"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 )
 
@@ -233,13 +234,36 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 
 	}
 
-	vregroot, err := address.NewIDAddress(80)
+	switch template.VerifregRootKey.Type {
-	if err != nil {
+	case genesis.TAccount:
-		return nil, nil, err
+		var ainfo genesis.AccountMeta
-	}
+		if err := json.Unmarshal(template.VerifregRootKey.Meta, &ainfo); err != nil {
+			return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+		}
+		st, err := cst.Put(ctx, &account0.State{Address: ainfo.Owner})
+		if err != nil {
+			return nil, nil, err
+		}
 
-	if err = createMultisigAccount(ctx, bs, cst, state, vregroot, template.VerifregRootKey, keyIDs); err != nil {
+		_, ok := keyIDs[ainfo.Owner]
-		return nil, nil, xerrors.Errorf("failed to set up verified registry signer: %w", err)
+		if ok {
+			return nil, nil, fmt.Errorf("rootkey account has already been declared, cannot be assigned 80: %s", ainfo.Owner)
+		}
+
+		err = state.SetActor(builtin.RootVerifierAddress, &types.Actor{
+			Code:    builtin0.AccountActorCodeID,
+			Balance: template.VerifregRootKey.Balance,
+			Head:    st,
+		})
+		if err != nil {
+			return nil, nil, xerrors.Errorf("setting verifreg rootkey account: %w", err)
+		}
+	case genesis.TMultisig:
+		if err = createMultisigAccount(ctx, bs, cst, state, builtin.RootVerifierAddress, template.VerifregRootKey, keyIDs); err != nil {
+			return nil, nil, xerrors.Errorf("failed to set up verified registry signer: %w", err)
+		}
+	default:
+		return nil, nil, xerrors.Errorf("unknown account type for verifreg rootkey: %w", err)
 	}
 
 	// Setup the first verifier as ID-address 81
@@ -300,8 +324,36 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
 
 	template.RemainderAccount.Balance = remainingFil
 
-	if err := createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil {
+	switch template.RemainderAccount.Type {
-		return nil, nil, xerrors.Errorf("failed to set up remainder account: %w", err)
+	case genesis.TAccount:
+		var ainfo genesis.AccountMeta
+		if err := json.Unmarshal(template.RemainderAccount.Meta, &ainfo); err != nil {
+			return nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err)
+		}
+		st, err := cst.Put(ctx, &account0.State{Address: ainfo.Owner})
+		if err != nil {
+			return nil, nil, err
+		}
+
+		_, ok := keyIDs[ainfo.Owner]
+		if ok {
+			return nil, nil, fmt.Errorf("remainder account has already been declared, cannot be assigned 90: %s", ainfo.Owner)
+		}
+
+		err = state.SetActor(builtin.ReserveAddress, &types.Actor{
+			Code:    builtin0.AccountActorCodeID,
+			Balance: template.RemainderAccount.Balance,
+			Head:    st,
+		})
+		if err != nil {
+			return nil, nil, xerrors.Errorf("setting remainder account: %w", err)
+		}
+	case genesis.TMultisig:
+		if err = createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil {
+			return nil, nil, xerrors.Errorf("failed to set up remainder: %w", err)
+		}
+	default:
+		return nil, nil, xerrors.Errorf("unknown account type for remainder: %w", err)
 	}
 
 	return state, keyIDs, nil
@@ -406,7 +458,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
 		StateBase:      stateroot,
 		Epoch:          0,
 		Rand:           &fakeRand{},
-		Bstore:         cs.Blockstore(),
+		Bstore:         cs.StateBlockstore(),
 		Syscalls:       mkFakedSigSyscalls(cs.VMSys()),
 		CircSupplyCalc: nil,
 		NtwkVersion:    genesisNetworkVersion,

chain/gen/genesis/miners.go

@@ -70,7 +70,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 		StateBase:      sroot,
 		Epoch:          0,
 		Rand:           &fakeRand{},
-		Bstore:         cs.Blockstore(),
+		Bstore:         cs.StateBlockstore(),
 		Syscalls:       mkFakedSigSyscalls(cs.VMSys()),
 		CircSupplyCalc: csc,
 		NtwkVersion:    genesisNetworkVersion,

chain/gen/mining.go

@@ -79,7 +79,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA
 		}
 	}
 
-	store := sm.ChainStore().Store(ctx)
+	store := sm.ChainStore().ActorStore(ctx)
 	blsmsgroot, err := toArray(store, blsMsgCids)
 	if err != nil {
 		return nil, xerrors.Errorf("building bls amt: %w", err)

chain/market/cbor_gen.go

@@ -5,12 +5,16 @@ package market
 import (
 	"fmt"
 	"io"
+	"sort"
 
+	cid "github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )
 
 var _ = xerrors.Errorf
+var _ = cid.Undef
+var _ = sort.Sort
 
 var lengthBufFundedAddressState = []byte{131}
 

chain/stmgr/call.go

@@ -59,7 +59,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 		StateBase:      bstate,
 		Epoch:          bheight,
 		Rand:           store.NewChainRand(sm.cs, ts.Cids()),
-		Bstore:         sm.cs.Blockstore(),
+		Bstore:         sm.cs.StateBlockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NtwkVersion:    sm.GetNtwkVersion,
@@ -174,7 +174,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		StateBase:      state,
 		Epoch:          ts.Height() + 1,
 		Rand:           r,
-		Bstore:         sm.cs.Blockstore(),
+		Bstore:         sm.cs.StateBlockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NtwkVersion:    sm.GetNtwkVersion,

chain/stmgr/forks.go

@@ -15,6 +15,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/network"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -24,8 +25,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
-	"github.com/filecoin-project/lotus/lib/bufbstore"
 	builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
 	miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig"
@@ -422,20 +421,9 @@ func doTransfer(tree types.StateTree, from, to address.Address, amt abi.TokenAmo
 	if cb != nil {
 		// record the transfer in execution traces
 
-		fakeMsg := &types.Message{
-			From:  from,
-			To:    to,
-			Value: amt,
-		}
-		fakeRct := &types.MessageReceipt{
-			ExitCode: 0,
-			Return:   nil,
-			GasUsed:  0,
-		}
-
 		cb(types.ExecutionTrace{
-			Msg:        fakeMsg,
+			Msg:        makeFakeMsg(from, to, amt, 0),
-			MsgRct:     fakeRct,
+			MsgRct:     makeFakeRct(),
 			Error:      "",
 			Duration:   0,
 			GasCharges: nil,
@@ -516,7 +504,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 			}
 		case builtin0.StorageMinerActorCodeID:
 			var st miner0.State
-			if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil {
+			if err := sm.ChainStore().ActorStore(ctx).Get(ctx, act.Head, &st); err != nil {
 				return xerrors.Errorf("failed to load miner state: %w", err)
 			}
 
@@ -560,7 +548,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 		return cid.Undef, xerrors.Errorf("failed to load power actor: %w", err)
 	}
 
-	cst := cbor.NewCborStore(sm.ChainStore().Blockstore())
+	cst := cbor.NewCborStore(sm.ChainStore().StateBlockstore())
 	if err := cst.Get(ctx, powAct.Head, &ps); err != nil {
 		return cid.Undef, xerrors.Errorf("failed to get power actor state: %w", err)
 	}
@@ -594,7 +582,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 			}
 		case builtin0.StorageMinerActorCodeID:
 			var st miner0.State
-			if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil {
+			if err := sm.ChainStore().ActorStore(ctx).Get(ctx, act.Head, &st); err != nil {
 				return xerrors.Errorf("failed to load miner state: %w", err)
 			}
 
@@ -603,7 +591,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 				return xerrors.Errorf("failed to get miner info: %w", err)
 			}
 
-			sectorsArr, err := adt0.AsArray(sm.ChainStore().Store(ctx), st.Sectors)
+			sectorsArr, err := adt0.AsArray(sm.ChainStore().ActorStore(ctx), st.Sectors)
 			if err != nil {
 				return xerrors.Errorf("failed to load sectors array: %w", err)
 			}
@@ -623,11 +611,11 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 			lbact, err := lbtree.GetActor(addr)
 			if err == nil {
 				var lbst miner0.State
-				if err := sm.ChainStore().Store(ctx).Get(ctx, lbact.Head, &lbst); err != nil {
+				if err := sm.ChainStore().ActorStore(ctx).Get(ctx, lbact.Head, &lbst); err != nil {
 					return xerrors.Errorf("failed to load miner state: %w", err)
 				}
 
-				lbsectors, err := adt0.AsArray(sm.ChainStore().Store(ctx), lbst.Sectors)
+				lbsectors, err := adt0.AsArray(sm.ChainStore().ActorStore(ctx), lbst.Sectors)
 				if err != nil {
 					return xerrors.Errorf("failed to load lb sectors array: %w", err)
 				}
@@ -699,24 +687,14 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 	if cb != nil {
 		// record the transfer in execution traces
 
-		fakeMsg := &types.Message{
+		fakeMsg := makeFakeMsg(builtin.SystemActorAddr, builtin.SystemActorAddr, big.Zero(), uint64(epoch))
-			From:  builtin.SystemActorAddr,
-			To:    builtin.SystemActorAddr,
-			Value: big.Zero(),
-			Nonce: uint64(epoch),
-		}
-		fakeRct := &types.MessageReceipt{
-			ExitCode: 0,
-			Return:   nil,
-			GasUsed:  0,
-		}
 
 		if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{
-			MessageReceipt: *fakeRct,
+			MessageReceipt: *makeFakeRct(),
 			ActorErr:       nil,
 			ExecutionTrace: types.ExecutionTrace{
 				Msg:        fakeMsg,
-				MsgRct:     fakeRct,
+				MsgRct:     makeFakeRct(),
 				Error:      "",
 				Duration:   0,
 				GasCharges: nil,
@@ -733,7 +711,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, _ Migratio
 }
 
 func UpgradeIgnition(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
-	store := sm.cs.Store(ctx)
+	store := sm.cs.ActorStore(ctx)
 
 	if build.UpgradeLiftoffHeight <= epoch {
 		return cid.Undef, xerrors.Errorf("liftoff height must be beyond ignition height")
@@ -789,7 +767,7 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, _ MigrationCache, cb
 
 func UpgradeRefuel(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
 
-	store := sm.cs.Store(ctx)
+	store := sm.cs.ActorStore(ctx)
 	tree, err := sm.StateTree(root)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
@@ -814,7 +792,7 @@ func UpgradeRefuel(ctx context.Context, sm *StateManager, _ MigrationCache, cb E
 }
 
 func UpgradeActorsV2(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
-	buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync())
+	buf := blockstore.NewTieredBstore(sm.cs.StateBlockstore(), blockstore.NewMemorySync())
 	store := store.ActorStore(ctx, buf)
 
 	info, err := store.Put(ctx, new(types.StateInfo0))
@@ -865,7 +843,7 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, _ MigrationCache, cb
 		return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
 	}
 
-	err = setNetworkName(ctx, sm.cs.Store(ctx), tree, "mainnet")
+	err = setNetworkName(ctx, sm.cs.ActorStore(ctx), tree, "mainnet")
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("setting network name: %w", err)
 	}
@@ -874,7 +852,7 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, _ MigrationCache, cb
 }
 
 func UpgradeCalico(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
-	store := sm.cs.Store(ctx)
+	store := sm.cs.ActorStore(ctx)
 	var stateRoot types.StateRoot
 	if err := store.Get(ctx, root, &stateRoot); err != nil {
 		return cid.Undef, xerrors.Errorf("failed to decode state root: %w", err)
@@ -915,6 +893,66 @@ func UpgradeCalico(ctx context.Context, sm *StateManager, _ MigrationCache, cb E
 	return newRoot, nil
 }
 
+func terminateActor(ctx context.Context, tree *state.StateTree, addr address.Address, cb ExecCallback, epoch abi.ChainEpoch) error {
+	a, err := tree.GetActor(addr)
+	if xerrors.Is(err, types.ErrActorNotFound) {
+		return types.ErrActorNotFound
+	} else if err != nil {
+		return xerrors.Errorf("failed to get actor to delete: %w", err)
+	}
+
+	var trace types.ExecutionTrace
+	if err := doTransfer(tree, addr, builtin.BurntFundsActorAddr, a.Balance, func(t types.ExecutionTrace) {
+		trace = t
+	}); err != nil {
+		return xerrors.Errorf("transferring terminated actor's balance: %w", err)
+	}
+
+	if cb != nil {
+		// record the transfer in execution traces
+
+		fakeMsg := makeFakeMsg(builtin.SystemActorAddr, addr, big.Zero(), uint64(epoch))
+
+		if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{
+			MessageReceipt: *makeFakeRct(),
+			ActorErr:       nil,
+			ExecutionTrace: trace,
+			Duration:       0,
+			GasCosts:       nil,
+		}); err != nil {
+			return xerrors.Errorf("recording transfers: %w", err)
+		}
+	}
+
+	err = tree.DeleteActor(addr)
+	if err != nil {
+		return xerrors.Errorf("deleting actor from tree: %w", err)
+	}
+
+	ia, err := tree.GetActor(init_.Address)
+	if err != nil {
+		return xerrors.Errorf("loading init actor: %w", err)
+	}
+
+	ias, err := init_.Load(&state.AdtStore{IpldStore: tree.Store}, ia)
+	if err != nil {
+		return xerrors.Errorf("loading init actor state: %w", err)
+	}
+
+	if err := ias.Remove(addr); err != nil {
+		return xerrors.Errorf("deleting entry from address map: %w", err)
+	}
+
+	nih, err := tree.Store.Put(ctx, ias)
+	if err != nil {
+		return xerrors.Errorf("writing new init actor state: %w", err)
+	}
+
+	ia.Head = nih
+
+	return tree.SetActor(init_.Address, ia)
+}
+
 func UpgradeActorsV3(ctx context.Context, sm *StateManager, cache MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
 	// Use all the CPUs except 3.
 	workerCount := runtime.NumCPU() - 3
@@ -933,16 +971,21 @@ func UpgradeActorsV3(ctx context.Context, sm *StateManager, cache MigrationCache
 		return cid.Undef, xerrors.Errorf("migrating actors v3 state: %w", err)
 	}
 
-	// perform some basic sanity checks to make sure everything still works.
+	tree, err := sm.StateTree(newRoot)
-	store := store.ActorStore(ctx, sm.ChainStore().Blockstore())
+	if err != nil {
-	if newSm, err := state.LoadStateTree(store, newRoot); err != nil {
+		return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
-		return cid.Undef, xerrors.Errorf("state tree sanity load failed: %w", err)
+	}
-	} else if newRoot2, err := newSm.Flush(ctx); err != nil {
+
-		return cid.Undef, xerrors.Errorf("state tree sanity flush failed: %w", err)
+	if build.BuildType == build.BuildMainnet {
-	} else if newRoot2 != newRoot {
+		err := terminateActor(ctx, tree, build.ZeroAddress, cb, epoch)
-		return cid.Undef, xerrors.Errorf("state-root mismatch: %s != %s", newRoot, newRoot2)
+		if err != nil && !xerrors.Is(err, types.ErrActorNotFound) {
-	} else if _, err := newSm.GetActor(init_.Address); err != nil {
+			return cid.Undef, xerrors.Errorf("deleting zero bls actor: %w", err)
-		return cid.Undef, xerrors.Errorf("failed to load init actor after upgrade: %w", err)
+		}
+
+		newRoot, err = tree.Flush(ctx)
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("flushing state tree: %w", err)
+		}
 	}
 
 	return newRoot, nil
@@ -966,7 +1009,7 @@ func upgradeActorsV3Common(
 	root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
 	config nv10.Config,
 ) (cid.Cid, error) {
-	buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync())
+	buf := blockstore.NewTieredBstore(sm.cs.StateBlockstore(), blockstore.NewMemorySync())
 	store := store.ActorStore(ctx, buf)
 
 	// Load the state root.
@@ -1139,24 +1182,14 @@ func splitGenesisMultisig0(ctx context.Context, cb ExecCallback, addr address.Ad
 	if cb != nil {
 		// record the transfer in execution traces
 
-		fakeMsg := &types.Message{
+		fakeMsg := makeFakeMsg(builtin.SystemActorAddr, addr, big.Zero(), uint64(epoch))
-			From:  builtin.SystemActorAddr,
-			To:    addr,
-			Value: big.Zero(),
-			Nonce: uint64(epoch),
-		}
-		fakeRct := &types.MessageReceipt{
-			ExitCode: 0,
-			Return:   nil,
-			GasUsed:  0,
-		}
 
 		if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{
-			MessageReceipt: *fakeRct,
+			MessageReceipt: *makeFakeRct(),
 			ActorErr:       nil,
 			ExecutionTrace: types.ExecutionTrace{
 				Msg:        fakeMsg,
-				MsgRct:     fakeRct,
+				MsgRct:     makeFakeRct(),
 				Error:      "",
 				Duration:   0,
 				GasCharges: nil,
@@ -1206,7 +1239,7 @@ func resetGenesisMsigs0(ctx context.Context, sm *StateManager, store adt0.Store,
 		return xerrors.Errorf("getting genesis tipset: %w", err)
 	}
 
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	genesisTree, err := state.LoadStateTree(cst, gts.ParentState())
 	if err != nil {
 		return xerrors.Errorf("loading state tree: %w", err)
@@ -1275,3 +1308,20 @@ func resetMultisigVesting0(ctx context.Context, store adt0.Store, tree *state.St
 
 	return nil
 }
+
+func makeFakeMsg(from address.Address, to address.Address, amt abi.TokenAmount, nonce uint64) *types.Message {
+	return &types.Message{
+		From:  from,
+		To:    to,
+		Value: amt,
+		Nonce: nonce,
+	}
+}
+
+func makeFakeRct() *types.MessageReceipt {
+	return &types.MessageReceipt{
+		ExitCode: 0,
+		Return:   nil,
+		GasUsed:  0,
+	}
+}

chain/stmgr/forks_test.go

@@ -125,7 +125,7 @@ func TestForkHeightTriggers(t *testing.T) {
 			Height:  testForkHeight,
 			Migration: func(ctx context.Context, sm *StateManager, cache MigrationCache, cb ExecCallback,
 				root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
-				cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore())
+				cst := ipldcbor.NewCborStore(sm.ChainStore().StateBlockstore())
 
 				st, err := sm.StateTree(root)
 				if err != nil {

chain/stmgr/read.go

@@ -22,7 +22,7 @@ func (sm *StateManager) ParentStateTsk(tsk types.TipSetKey) (*state.StateTree, e
 }
 
 func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error) {
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	state, err := state.LoadStateTree(cst, sm.parentState(ts))
 	if err != nil {
 		return nil, xerrors.Errorf("load state tree: %w", err)
@@ -32,7 +32,7 @@ func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error)
 }
 
 func (sm *StateManager) StateTree(st cid.Cid) (*state.StateTree, error) {
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	state, err := state.LoadStateTree(cst, st)
 	if err != nil {
 		return nil, xerrors.Errorf("load state tree: %w", err)

chain/stmgr/stmgr.go

@@ -286,7 +286,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 			StateBase:      base,
 			Epoch:          epoch,
 			Rand:           r,
-			Bstore:         sm.cs.Blockstore(),
+			Bstore:         sm.cs.StateBlockstore(),
 			Syscalls:       sm.cs.VMSys(),
 			CircSupplyCalc: sm.GetVMCirculatingSupply,
 			NtwkVersion:    sm.GetNtwkVersion,
@@ -430,7 +430,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
 		return cid.Cid{}, cid.Cid{}, err
 	}
 
-	rectarr := blockadt.MakeEmptyArray(sm.cs.Store(ctx))
+	rectarr := blockadt.MakeEmptyArray(sm.cs.ActorStore(ctx))
 	for i, receipt := range receipts {
 		if err := rectarr.Set(uint64(i), receipt); err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("failed to build receipts amt: %w", err)
@@ -515,7 +515,7 @@ func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Ad
 		ts = sm.cs.GetHeaviestTipSet()
 	}
 
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 
 	// First try to resolve the actor in the parent state, so we don't have to compute anything.
 	tree, err := state.LoadStateTree(cst, ts.ParentState())
@@ -556,7 +556,7 @@ func (sm *StateManager) GetBlsPublicKey(ctx context.Context, addr address.Addres
 }
 
 func (sm *StateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	state, err := state.LoadStateTree(cst, sm.parentState(ts))
 	if err != nil {
 		return address.Undef, xerrors.Errorf("load state tree: %w", err)
@@ -882,7 +882,7 @@ func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address,
 		return api.MarketBalance{}, err
 	}
 
-	mstate, err := market.Load(sm.cs.Store(ctx), act)
+	mstate, err := market.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return api.MarketBalance{}, err
 	}
@@ -966,7 +966,7 @@ func (sm *StateManager) setupGenesisVestingSchedule(ctx context.Context) error {
 		return xerrors.Errorf("getting genesis tipset state: %w", err)
 	}
 
-	cst := cbor.NewCborStore(sm.cs.Blockstore())
+	cst := cbor.NewCborStore(sm.cs.StateBlockstore())
 	sTree, err := state.LoadStateTree(cst, st)
 	if err != nil {
 		return xerrors.Errorf("loading state tree: %w", err)
@@ -1325,7 +1325,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
 			unCirc = big.Add(unCirc, actor.Balance)
 
 		case a == market.Address:
-			mst, err := market.Load(sm.cs.Store(ctx), actor)
+			mst, err := market.Load(sm.cs.ActorStore(ctx), actor)
 			if err != nil {
 				return err
 			}
@@ -1342,7 +1342,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
 			circ = big.Add(circ, actor.Balance)
 
 		case builtin.IsStorageMinerActor(actor.Code):
-			mst, err := miner.Load(sm.cs.Store(ctx), actor)
+			mst, err := miner.Load(sm.cs.ActorStore(ctx), actor)
 			if err != nil {
 				return err
 			}
@@ -1359,7 +1359,7 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
 			}
 
 		case builtin.IsMultisigActor(actor.Code):
-			mst, err := multisig.Load(sm.cs.Store(ctx), actor)
+			mst, err := multisig.Load(sm.cs.ActorStore(ctx), actor)
 			if err != nil {
 				return err
 			}
@@ -1413,7 +1413,7 @@ func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address,
 		return nil, nil, err
 	}
 
-	actState, err := paych.Load(sm.cs.Store(ctx), act)
+	actState, err := paych.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -1431,7 +1431,7 @@ func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (m
 		return nil, err
 	}
 
-	actState, err := market.Load(sm.cs.Store(ctx), act)
+	actState, err := market.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, err
 	}

chain/stmgr/utils.go

@@ -48,7 +48,7 @@ func GetNetworkName(ctx context.Context, sm *StateManager, st cid.Cid) (dtypes.N
 	if err != nil {
 		return "", err
 	}
-	ias, err := init_.Load(sm.cs.Store(ctx), act)
+	ias, err := init_.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return "", err
 	}
@@ -65,7 +65,7 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr
 	if err != nil {
 		return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
 	}
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
 	}
@@ -75,7 +75,7 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr
 		return address.Undef, xerrors.Errorf("failed to load actor info: %w", err)
 	}
 
-	return vm.ResolveToKeyAddr(state, sm.cs.Store(ctx), info.Worker)
+	return vm.ResolveToKeyAddr(state, sm.cs.ActorStore(ctx), info.Worker)
 }
 
 func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (power.Claim, power.Claim, bool, error) {
@@ -88,7 +88,7 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres
 		return power.Claim{}, power.Claim{}, false, xerrors.Errorf("(get sset) failed to load power actor state: %w", err)
 	}
 
-	pas, err := power.Load(sm.cs.Store(ctx), act)
+	pas, err := power.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return power.Claim{}, power.Claim{}, false, err
 	}
@@ -123,7 +123,7 @@ func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address,
 		return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
 	}
 
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
 	}
@@ -137,7 +137,7 @@ func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Addres
 		return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
 	}
 
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
 	}
@@ -151,7 +151,7 @@ func GetSectorsForWinningPoSt(ctx context.Context, nv network.Version, pv ffiwra
 		return nil, xerrors.Errorf("failed to load miner actor: %w", err)
 	}
 
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
 	}
@@ -249,7 +249,7 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma
 		return false, xerrors.Errorf("failed to load power actor: %w", err)
 	}
 
-	spas, err := power.Load(sm.cs.Store(ctx), act)
+	spas, err := power.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return false, xerrors.Errorf("failed to load power actor state: %w", err)
 	}
@@ -272,7 +272,7 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts
 		return nil, xerrors.Errorf("failed to load market actor: %w", err)
 	}
 
-	state, err := market.Load(sm.cs.Store(ctx), act)
+	state, err := market.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load market actor state: %w", err)
 	}
@@ -320,7 +320,7 @@ func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([
 		return nil, xerrors.Errorf("failed to load power actor: %w", err)
 	}
 
-	powState, err := power.Load(sm.cs.Store(ctx), act)
+	powState, err := power.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load power actor state: %w", err)
 	}
@@ -353,7 +353,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		StateBase:      base,
 		Epoch:          height,
 		Rand:           r,
-		Bstore:         sm.cs.Blockstore(),
+		Bstore:         sm.cs.StateBlockstore(),
 		Syscalls:       sm.cs.VMSys(),
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
 		NtwkVersion:    sm.GetNtwkVersion,
@@ -474,7 +474,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule
 		return nil, xerrors.Errorf("failed to load miner actor: %w", err)
 	}
 
-	mas, err := miner.Load(sm.cs.Store(ctx), act)
+	mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
 	}
@@ -623,7 +623,7 @@ func minerHasMinPower(ctx context.Context, sm *StateManager, addr address.Addres
 		return false, xerrors.Errorf("loading power actor state: %w", err)
 	}
 
-	ps, err := power.Load(sm.cs.Store(ctx), pact)
+	ps, err := power.Load(sm.cs.ActorStore(ctx), pact)
 	if err != nil {
 		return false, err
 	}
@@ -654,7 +654,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
 		return false, xerrors.Errorf("loading power actor state: %w", err)
 	}
 
-	pstate, err := power.Load(sm.cs.Store(ctx), pact)
+	pstate, err := power.Load(sm.cs.ActorStore(ctx), pact)
 	if err != nil {
 		return false, err
 	}
@@ -664,7 +664,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
 		return false, xerrors.Errorf("loading miner actor state: %w", err)
 	}
 
-	mstate, err := miner.Load(sm.cs.Store(ctx), mact)
+	mstate, err := miner.Load(sm.cs.ActorStore(ctx), mact)
 	if err != nil {
 		return false, err
 	}
@@ -696,7 +696,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
 }
 
 func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi.TokenAmount, error) {
-	str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState())
+	str, err := state.LoadStateTree(sm.ChainStore().ActorStore(ctx), ts.ParentState())
 	if err != nil {
 		return abi.TokenAmount{}, err
 	}

chain/store/index_test.go

@@ -6,10 +6,10 @@ import (
 	"testing"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types/mock"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	datastore "github.com/ipfs/go-datastore"
 	syncds "github.com/ipfs/go-datastore/sync"
 	"github.com/stretchr/testify/assert"
@@ -30,7 +30,7 @@ func TestIndexSeeks(t *testing.T) {
 
 	ctx := context.TODO()
 
-	nbs := blockstore.NewTemporarySync()
+	nbs := blockstore.NewMemorySync()
 	cs := store.NewChainStore(nbs, nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil)
 	defer cs.Close() //nolint:errcheck
 

chain/store/store.go

@@ -23,12 +23,12 @@ import (
 	blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
 
 	"github.com/filecoin-project/lotus/api"
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/journal"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/metrics"
 
 	"go.opencensus.io/stats"
@@ -81,7 +81,7 @@ func init() {
 }
 
 // ReorgNotifee represents a callback that gets called upon reorgs.
-type ReorgNotifee func(rev, app []*types.TipSet) error
+type ReorgNotifee = func(rev, app []*types.TipSet) error
 
 // Journal event types.
 const (
@@ -107,11 +107,11 @@ type HeadChangeEvt struct {
 //   1. a tipset cache
 //   2. a block => messages references cache.
 type ChainStore struct {
-	bs      bstore.Blockstore
+	chainBlockstore bstore.Blockstore
-	localbs bstore.Blockstore
+	stateBlockstore bstore.Blockstore
-	ds      dstore.Batching
+	metadataDs      dstore.Batching
 
-	localviewer bstore.Viewer
+	chainLocalBlockstore bstore.Blockstore
 
 	heaviestLk sync.Mutex
 	heaviest   *types.TipSet
@@ -139,30 +139,29 @@ type ChainStore struct {
 	wg       sync.WaitGroup
 }
 
-// localbs is guaranteed to fail Get* if requested block isn't stored locally
+func NewChainStore(chainBs bstore.Blockstore, stateBs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
-func NewChainStore(bs bstore.Blockstore, localbs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
+	c, _ := lru.NewARC(DefaultMsgMetaCacheSize)
-	mmCache, _ := lru.NewARC(DefaultMsgMetaCacheSize)
+	tsc, _ := lru.NewARC(DefaultTipSetCacheSize)
-	tsCache, _ := lru.NewARC(DefaultTipSetCacheSize)
 	if j == nil {
 		j = journal.NilJournal()
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
+	// unwraps the fallback store in case one is configured.
+	// some methods _need_ to operate on a local blockstore only.
+	localbs, _ := bstore.UnwrapFallbackStore(chainBs)
 	cs := &ChainStore{
-		bs:       bs,
+		chainBlockstore:      chainBs,
-		localbs:  localbs,
+		stateBlockstore:      stateBs,
-		ds:       ds,
+		chainLocalBlockstore: localbs,
-		bestTips: pubsub.New(64),
+		metadataDs:           ds,
-		tipsets:  make(map[abi.ChainEpoch][]cid.Cid),
+		bestTips:             pubsub.New(64),
-		mmCache:  mmCache,
+		tipsets:              make(map[abi.ChainEpoch][]cid.Cid),
-		tsCache:  tsCache,
+		mmCache:              c,
-		vmcalls:  vmcalls,
+		tsCache:              tsc,
-		cancelFn: cancel,
+		vmcalls:              vmcalls,
-		journal:  j,
+		cancelFn:             cancel,
-	}
+		journal:              j,
-
-	if v, ok := localbs.(bstore.Viewer); ok {
-		cs.localviewer = v
 	}
 
 	cs.evtTypes = [1]journal.EventType{
@@ -216,7 +215,7 @@ func (cs *ChainStore) Close() error {
 }
 
 func (cs *ChainStore) Load() error {
-	head, err := cs.ds.Get(chainHeadKey)
+	head, err := cs.metadataDs.Get(chainHeadKey)
 	if err == dstore.ErrNotFound {
 		log.Warn("no previous chain state found")
 		return nil
@@ -246,7 +245,7 @@ func (cs *ChainStore) writeHead(ts *types.TipSet) error {
 		return xerrors.Errorf("failed to marshal tipset: %w", err)
 	}
 
-	if err := cs.ds.Put(chainHeadKey, data); err != nil {
+	if err := cs.metadataDs.Put(chainHeadKey, data); err != nil {
 		return xerrors.Errorf("failed to write chain head to datastore: %w", err)
 	}
 
@@ -306,13 +305,13 @@ func (cs *ChainStore) SubscribeHeadChanges(f ReorgNotifee) {
 func (cs *ChainStore) IsBlockValidated(ctx context.Context, blkid cid.Cid) (bool, error) {
 	key := blockValidationCacheKeyPrefix.Instance(blkid.String())
 
-	return cs.ds.Has(key)
+	return cs.metadataDs.Has(key)
 }
 
 func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
 	key := blockValidationCacheKeyPrefix.Instance(blkid.String())
 
-	if err := cs.ds.Put(key, []byte{0}); err != nil {
+	if err := cs.metadataDs.Put(key, []byte{0}); err != nil {
 		return xerrors.Errorf("cache block validation: %w", err)
 	}
 
@@ -322,7 +321,7 @@ func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) e
 func (cs *ChainStore) UnmarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
 	key := blockValidationCacheKeyPrefix.Instance(blkid.String())
 
-	if err := cs.ds.Delete(key); err != nil {
+	if err := cs.metadataDs.Delete(key); err != nil {
 		return xerrors.Errorf("removing from valid block cache: %w", err)
 	}
 
@@ -339,7 +338,7 @@ func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error {
 		return err
 	}
 
-	return cs.ds.Put(dstore.NewKey("0"), b.Cid().Bytes())
+	return cs.metadataDs.Put(dstore.NewKey("0"), b.Cid().Bytes())
 }
 
 func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error {
@@ -594,7 +593,7 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet)
 // FlushValidationCache removes all results of block validation from the
 // chain metadata store. Usually the first step after a new chain import.
 func (cs *ChainStore) FlushValidationCache() error {
-	return FlushValidationCache(cs.ds)
+	return FlushValidationCache(cs.metadataDs)
 }
 
 func FlushValidationCache(ds datastore.Batching) error {
@@ -653,7 +652,7 @@ func (cs *ChainStore) SetHead(ts *types.TipSet) error {
 // Contains returns whether our BlockStore has all blocks in the supplied TipSet.
 func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) {
 	for _, c := range ts.Cids() {
-		has, err := cs.bs.Has(c)
+		has, err := cs.chainBlockstore.Has(c)
 		if err != nil {
 			return false, err
 		}
@@ -668,16 +667,8 @@ func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) {
 // GetBlock fetches a BlockHeader with the supplied CID. It returns
 // blockstore.ErrNotFound if the block was not found in the BlockStore.
 func (cs *ChainStore) GetBlock(c cid.Cid) (*types.BlockHeader, error) {
-	if cs.localviewer == nil {
-		sb, err := cs.localbs.Get(c)
-		if err != nil {
-			return nil, err
-		}
-		return types.DecodeBlock(sb.RawData())
-	}
-
 	var blk *types.BlockHeader
-	err := cs.localviewer.View(c, func(b []byte) (err error) {
+	err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
 		blk, err = types.DecodeBlock(b)
 		return err
 	})
@@ -851,7 +842,7 @@ func (cs *ChainStore) PersistBlockHeaders(b ...*types.BlockHeader) error {
 			end = len(b)
 		}
 
-		err = multierr.Append(err, cs.bs.PutMany(sbs[start:end]))
+		err = multierr.Append(err, cs.chainLocalBlockstore.PutMany(sbs[start:end]))
 	}
 
 	return err
@@ -875,7 +866,7 @@ func PutMessage(bs bstore.Blockstore, m storable) (cid.Cid, error) {
 }
 
 func (cs *ChainStore) PutMessage(m storable) (cid.Cid, error) {
-	return PutMessage(cs.bs, m)
+	return PutMessage(cs.chainBlockstore, m)
 }
 
 func (cs *ChainStore) expandTipset(b *types.BlockHeader) (*types.TipSet, error) {
@@ -936,7 +927,7 @@ func (cs *ChainStore) AddBlock(ctx context.Context, b *types.BlockHeader) error
 }
 
 func (cs *ChainStore) GetGenesis() (*types.BlockHeader, error) {
-	data, err := cs.ds.Get(dstore.NewKey("0"))
+	data, err := cs.metadataDs.Get(dstore.NewKey("0"))
 	if err != nil {
 		return nil, err
 	}
@@ -962,17 +953,8 @@ func (cs *ChainStore) GetCMessage(c cid.Cid) (types.ChainMsg, error) {
 }
 
 func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) {
-	if cs.localviewer == nil {
-		sb, err := cs.localbs.Get(c)
-		if err != nil {
-			log.Errorf("get message get failed: %s: %s", c, err)
-			return nil, err
-		}
-		return types.DecodeMessage(sb.RawData())
-	}
-
 	var msg *types.Message
-	err := cs.localviewer.View(c, func(b []byte) (err error) {
+	err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
 		msg, err = types.DecodeMessage(b)
 		return err
 	})
@@ -980,17 +962,8 @@ func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) {
 }
 
 func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error) {
-	if cs.localviewer == nil {
-		sb, err := cs.localbs.Get(c)
-		if err != nil {
-			log.Errorf("get message get failed: %s: %s", c, err)
-			return nil, err
-		}
-		return types.DecodeSignedMessage(sb.RawData())
-	}
-
 	var msg *types.SignedMessage
-	err := cs.localviewer.View(c, func(b []byte) (err error) {
+	err := cs.chainLocalBlockstore.View(c, func(b []byte) (err error) {
 		msg, err = types.DecodeSignedMessage(b)
 		return err
 	})
@@ -1000,7 +973,7 @@ func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error)
 func (cs *ChainStore) readAMTCids(root cid.Cid) ([]cid.Cid, error) {
 	ctx := context.TODO()
 	// block headers use adt0, for now.
-	a, err := blockadt.AsArray(cs.Store(ctx), root)
+	a, err := blockadt.AsArray(cs.ActorStore(ctx), root)
 	if err != nil {
 		return nil, xerrors.Errorf("amt load: %w", err)
 	}
@@ -1124,7 +1097,7 @@ func (cs *ChainStore) ReadMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error)
 		return mmcids.bls, mmcids.secpk, nil
 	}
 
-	cst := cbor.NewCborStore(cs.localbs)
+	cst := cbor.NewCborStore(cs.chainLocalBlockstore)
 	var msgmeta types.MsgMeta
 	if err := cst.Get(context.TODO(), mmc, &msgmeta); err != nil {
 		return nil, nil, xerrors.Errorf("failed to load msgmeta (%s): %w", mmc, err)
@@ -1194,7 +1167,7 @@ func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message,
 func (cs *ChainStore) GetParentReceipt(b *types.BlockHeader, i int) (*types.MessageReceipt, error) {
 	ctx := context.TODO()
 	// block headers use adt0, for now.
-	a, err := blockadt.AsArray(cs.Store(ctx), b.ParentMessageReceipts)
+	a, err := blockadt.AsArray(cs.ActorStore(ctx), b.ParentMessageReceipts)
 	if err != nil {
 		return nil, xerrors.Errorf("amt load: %w", err)
 	}
@@ -1237,16 +1210,26 @@ func (cs *ChainStore) LoadSignedMessagesFromCids(cids []cid.Cid) ([]*types.Signe
 	return msgs, nil
 }
 
-func (cs *ChainStore) Blockstore() bstore.Blockstore {
+// ChainBlockstore returns the chain blockstore. Currently the chain and state
-	return cs.bs
+// // stores are both backed by the same physical store, albeit with different
+// // caching policies, but in the future they will segregate.
+func (cs *ChainStore) ChainBlockstore() bstore.Blockstore {
+	return cs.chainBlockstore
+}
+
+// StateBlockstore returns the state blockstore. Currently the chain and state
+// stores are both backed by the same physical store, albeit with different
+// caching policies, but in the future they will segregate.
+func (cs *ChainStore) StateBlockstore() bstore.Blockstore {
+	return cs.stateBlockstore
 }
 
 func ActorStore(ctx context.Context, bs bstore.Blockstore) adt.Store {
 	return adt.WrapStore(ctx, cbor.NewCborStore(bs))
 }
 
-func (cs *ChainStore) Store(ctx context.Context) adt.Store {
+func (cs *ChainStore) ActorStore(ctx context.Context) adt.Store {
-	return ActorStore(ctx, cs.bs)
+	return ActorStore(ctx, cs.stateBlockstore)
 }
 
 func (cs *ChainStore) VMSys() vm.SyscallBuilder {
@@ -1444,8 +1427,9 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
 		return xerrors.Errorf("failed to write car header: %s", err)
 	}
 
-	return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, func(c cid.Cid) error {
+	unionBs := bstore.Union(cs.stateBlockstore, cs.chainBlockstore)
-		blk, err := cs.bs.Get(c)
+	return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, true, func(c cid.Cid) error {
+		blk, err := unionBs.Get(c)
 		if err != nil {
 			return xerrors.Errorf("writing object to car, bs.Get: %w", err)
 		}
@@ -1458,7 +1442,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
 	})
 }
 
-func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, cb func(cid.Cid) error) error {
+func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs, skipMsgReceipts bool, cb func(cid.Cid) error) error {
 	if ts == nil {
 		ts = cs.GetHeaviestTipSet()
 	}
@@ -1478,7 +1462,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 			return err
 		}
 
-		data, err := cs.bs.Get(blk)
+		data, err := cs.chainBlockstore.Get(blk)
 		if err != nil {
 			return xerrors.Errorf("getting block: %w", err)
 		}
@@ -1498,7 +1482,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 		var cids []cid.Cid
 		if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots {
 			if walked.Visit(b.Messages) {
-				mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages})
+				mcids, err := recurseLinks(cs.chainBlockstore, walked, b.Messages, []cid.Cid{b.Messages})
 				if err != nil {
 					return xerrors.Errorf("recursing messages failed: %w", err)
 				}
@@ -1519,13 +1503,17 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 
 		if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots {
 			if walked.Visit(b.ParentStateRoot) {
-				cids, err := recurseLinks(cs.bs, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
+				cids, err := recurseLinks(cs.stateBlockstore, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
 				if err != nil {
 					return xerrors.Errorf("recursing genesis state failed: %w", err)
 				}
 
 				out = append(out, cids...)
 			}
+
+			if !skipMsgReceipts && walked.Visit(b.ParentMessageReceipts) {
+				out = append(out, b.ParentMessageReceipts)
+			}
 		}
 
 		for _, c := range out {
@@ -1561,7 +1549,12 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 }
 
 func (cs *ChainStore) Import(r io.Reader) (*types.TipSet, error) {
-	header, err := car.LoadCar(cs.Blockstore(), r)
+	// TODO: writing only to the state blockstore is incorrect.
+	//  At this time, both the state and chain blockstores are backed by the
+	//  universal store. When we physically segregate the stores, we will need
+	//  to route state objects to the state blockstore, and chain objects to
+	//  the chain blockstore.
+	header, err := car.LoadCar(cs.StateBlockstore(), r)
 	if err != nil {
 		return nil, xerrors.Errorf("loadcar failed: %w", err)
 	}

chain/store/store_test.go

@@ -11,12 +11,12 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/crypto"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -52,7 +52,7 @@ func BenchmarkGetRandomness(b *testing.B) {
 		b.Fatal(err)
 	}
 
-	bs, err := lr.Blockstore(context.TODO(), repo.BlockstoreChain)
+	bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -104,7 +104,7 @@ func TestChainExportImport(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	nbs := blockstore.NewTemporary()
+	nbs := blockstore.NewMemory()
 	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
 	defer cs.Close() //nolint:errcheck
 
@@ -139,7 +139,7 @@ func TestChainExportImportFull(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	nbs := blockstore.NewTemporary()
+	nbs := blockstore.NewMemory()
 	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
 	defer cs.Close() //nolint:errcheck
 

chain/store/weight.go

@@ -28,7 +28,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
 
 	tpow := big2.Zero()
 	{
-		cst := cbor.NewCborStore(cs.Blockstore())
+		cst := cbor.NewCborStore(cs.StateBlockstore())
 		state, err := state.LoadStateTree(cst, ts.ParentState())
 		if err != nil {
 			return types.NewInt(0), xerrors.Errorf("load state tree: %w", err)
@@ -39,7 +39,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
 			return types.NewInt(0), xerrors.Errorf("get power actor: %w", err)
 		}
 
-		powState, err := power.Load(cs.Store(ctx), act)
+		powState, err := power.Load(cs.ActorStore(ctx), act)
 		if err != nil {
 			return types.NewInt(0), xerrors.Errorf("failed to load power actor state: %w", err)
 		}

chain/sub/incoming.go

@@ -7,13 +7,13 @@ import (
 	"time"
 
 	address "github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node/impl/client"
@@ -101,7 +101,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
 					[]tag.Mutator{tag.Insert(metrics.MinerID, blk.Header.Miner.String())},
 					metrics.BlockDelay.M(delay),
 				)
-				log.Warnf("Received block with large delay %d from miner %s", delay, blk.Header.Miner)
+				log.Warnw("received block with large delay from miner", "block", blk.Cid(), "delay", delay, "miner", blk.Header.Miner)
 			}
 
 			if s.InformNewBlock(msg.ReceivedFrom, &types.FullBlock{
@@ -392,7 +392,7 @@ func (bv *BlockValidator) isChainNearSynced() bool {
 func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockMsg) error {
 	// TODO there has to be a simpler way to do this without the blockstore dance
 	// block headers use adt0
-	store := blockadt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary()))
+	store := blockadt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewMemory()))
 	bmArr := blockadt.MakeEmptyArray(store)
 	smArr := blockadt.MakeEmptyArray(store)
 

chain/sync.go

@@ -44,6 +44,7 @@ import (
 	proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
 
 	"github.com/filecoin-project/lotus/api"
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/beacon"
@@ -54,7 +55,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/lib/sigs"
 	"github.com/filecoin-project/lotus/metrics"
 )
@@ -321,7 +321,7 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
 
 	// We use a temporary bstore here to avoid writing intermediate pieces
 	// into the blockstore.
-	blockstore := bstore.NewTemporary()
+	blockstore := bstore.NewMemory()
 	cst := cbor.NewCborStore(blockstore)
 
 	var bcids, scids []cid.Cid
@@ -354,7 +354,7 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
 	}
 
 	// Finally, flush.
-	return vm.Copy(context.TODO(), blockstore, syncer.store.Blockstore(), smroot)
+	return vm.Copy(context.TODO(), blockstore, syncer.store.ChainBlockstore(), smroot)
 }
 
 func (syncer *Syncer) LocalPeer() peer.ID {
@@ -640,7 +640,7 @@ func (syncer *Syncer) minerIsValid(ctx context.Context, maddr address.Address, b
 		return xerrors.Errorf("failed to load power actor: %w", err)
 	}
 
-	powState, err := power.Load(syncer.store.Store(ctx), act)
+	powState, err := power.Load(syncer.store.ActorStore(ctx), act)
 	if err != nil {
 		return xerrors.Errorf("failed to load power actor state: %w", err)
 	}
@@ -1055,7 +1055,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
 		return err
 	}
 
-	st, err := state.LoadStateTree(syncer.store.Store(ctx), stateroot)
+	st, err := state.LoadStateTree(syncer.store.ActorStore(ctx), stateroot)
 	if err != nil {
 		return xerrors.Errorf("failed to load base state tree: %w", err)
 	}
@@ -1102,7 +1102,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
 	}
 
 	// Validate message arrays in a temporary blockstore.
-	tmpbs := bstore.NewTemporary()
+	tmpbs := bstore.NewMemory()
 	tmpstore := blockadt.WrapStore(ctx, cbor.NewCborStore(tmpbs))
 
 	bmArr := blockadt.MakeEmptyArray(tmpstore)
@@ -1172,7 +1172,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock
 	}
 
 	// Finally, flush.
-	return vm.Copy(ctx, tmpbs, syncer.store.Blockstore(), mrcid)
+	return vm.Copy(ctx, tmpbs, syncer.store.ChainBlockstore(), mrcid)
 }
 
 func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signature, msgs []cid.Cid, pubks [][]byte) error {
@@ -1553,7 +1553,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
 
 		for bsi := 0; bsi < len(bstout); bsi++ {
 			// temp storage so we don't persist data we dont want to
-			bs := bstore.NewTemporary()
+			bs := bstore.NewMemory()
 			blks := cbor.NewCborStore(bs)
 
 			this := headers[i-bsi]
@@ -1574,7 +1574,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS
 				return err
 			}
 
-			if err := copyBlockstore(ctx, bs, syncer.store.Blockstore()); err != nil {
+			if err := copyBlockstore(ctx, bs, syncer.store.ChainBlockstore()); err != nil {
 				return xerrors.Errorf("message processing failed: %w", err)
 			}
 		}

chain/types/cbor_gen.go

@@ -5,6 +5,7 @@ package types
 import (
 	"fmt"
 	"io"
+	"sort"
 
 	abi "github.com/filecoin-project/go-state-types/abi"
 	crypto "github.com/filecoin-project/go-state-types/crypto"
@@ -16,6 +17,8 @@ import (
 )
 
 var _ = xerrors.Errorf
+var _ = cid.Undef
+var _ = sort.Sort
 
 var lengthBufBlockHeader = []byte{144}
 

chain/vm/mkactor.go

@@ -3,6 +3,10 @@ package vm
 import (
 	"context"
 
+	"github.com/filecoin-project/go-state-types/network"
+
+	"github.com/filecoin-project/lotus/build"
+
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/exitcode"
 	"github.com/filecoin-project/lotus/chain/actors"
@@ -39,6 +43,10 @@ func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, add
 		return nil, address.Undef, err
 	}
 
+	if addr == build.ZeroAddress && rt.NetworkVersion() >= network.Version10 {
+		return nil, address.Undef, aerrors.New(exitcode.ErrIllegalArgument, "cannot create the zero bls actor")
+	}
+
 	addrID, err := rt.state.RegisterNewAddress(addr)
 	if err != nil {
 		return nil, address.Undef, aerrors.Escalate(err, "registering actor address")

chain/vm/vm.go

@@ -28,6 +28,7 @@ import (
 	"github.com/filecoin-project/go-state-types/exitcode"
 	"github.com/filecoin-project/go-state-types/network"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/aerrors"
@@ -36,9 +37,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
-	bstore "github.com/filecoin-project/lotus/lib/blockstore"
-	"github.com/filecoin-project/lotus/lib/bufbstore"
 )
 
 const MaxCallDepth = 4096
@@ -208,7 +206,7 @@ type VM struct {
 	cstate         *state.StateTree
 	base           cid.Cid
 	cst            *cbor.BasicIpldStore
-	buf            *bufbstore.BufferedBS
+	buf            *blockstore.BufferedBlockstore
 	blockHeight    abi.ChainEpoch
 	areg           *ActorRegistry
 	rand           Rand
@@ -224,7 +222,7 @@ type VMOpts struct {
 	StateBase      cid.Cid
 	Epoch          abi.ChainEpoch
 	Rand           Rand
-	Bstore         bstore.Blockstore
+	Bstore         blockstore.Blockstore
 	Syscalls       SyscallBuilder
 	CircSupplyCalc CircSupplyCalculator
 	NtwkVersion    NtwkVersionGetter // TODO: stebalien: In what cases do we actually need this? It seems like even when creating new networks we want to use the 'global'/build-default version getter
@@ -233,7 +231,7 @@ type VMOpts struct {
 }
 
 func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) {
-	buf := bufbstore.NewBufferedBstore(opts.Bstore)
+	buf := blockstore.NewBuffered(opts.Bstore)
 	cst := cbor.NewCborStore(buf)
 	state, err := state.LoadStateTree(cst, opts.StateBase)
 	if err != nil {

cli/auth.go

@@ -9,6 +9,7 @@ import (
 	"github.com/filecoin-project/go-jsonrpc/auth"
 
 	"github.com/filecoin-project/lotus/api/apistruct"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -127,7 +128,7 @@ var authApiInfoToken = &cli.Command{
 
 		// TODO: Log in audit log when it is implemented
 
-		fmt.Printf("%s=%s:%s\n", envForRepo(t), string(token), ainfo.Addr)
+		fmt.Printf("%s=%s:%s\n", cliutil.EnvForRepo(t), string(token), ainfo.Addr)
 		return nil
 	},
 }

cli/backup.go

@@ -51,7 +51,10 @@ func BackupCmd(repoFlag string, rt repo.RepoType, getApi BackupApiFn) *cli.Comma
 			return xerrors.Errorf("getting metadata datastore: %w", err)
 		}
 
-		bds := backupds.Wrap(mds)
+		bds, err := backupds.Wrap(mds, backupds.NoLogdir)
+		if err != nil {
+			return err
+		}
 
 		fpath, err := homedir.Expand(cctx.Args().First())
 		if err != nil {

cli/chain.go

@@ -642,7 +642,10 @@ var chainListCmd = &cli.Command{
 						gasUsed += r.GasUsed
 					}
 
-					fmt.Printf("\ttipset: \t%d msgs, %d / %d (%0.2f%%)\n", len(msgs), gasUsed, limitSum, 100*float64(gasUsed)/float64(limitSum))
+					gasEfficiency := 100 * float64(gasUsed) / float64(limitSum)
+					gasCapacity := 100 * float64(limitSum) / float64(build.BlockGasLimit)
+
+					fmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
 				}
 				fmt.Println()
 			}

cli/client.go

@@ -1195,6 +1195,11 @@ var clientListAsksCmd = &cli.Command{
 		&cli.BoolFlag{
 			Name: "by-ping",
 		},
+		&cli.StringFlag{
+			Name:  "output-format",
+			Value: "text",
+			Usage: "Either 'text' or 'csv'",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
@@ -1214,11 +1219,16 @@ var clientListAsksCmd = &cli.Command{
 				return asks[i].Ping < asks[j].Ping
 			})
 		}
+		pfmt := "%s: min:%s max:%s price:%s/GiB/Epoch verifiedPrice:%s/GiB/Epoch ping:%s\n"
+		if cctx.String("output-format") == "csv" {
+			fmt.Printf("Miner,Min,Max,Price,VerifiedPrice,Ping\n")
+			pfmt = "%s,%s,%s,%s,%s,%s\n"
+		}
 
 		for _, a := range asks {
 			ask := a.Ask
 
-			fmt.Printf("%s: min:%s max:%s price:%s/GiB/Epoch verifiedPrice:%s/GiB/Epoch ping:%s\n", ask.Miner,
+			fmt.Printf(pfmt, ask.Miner,
 				types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))),
 				types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))),
 				types.FIL(ask.Price),
@@ -1237,7 +1247,13 @@ type QueriedAsk struct {
 }
 
 func GetAsks(ctx context.Context, api lapi.FullNode) ([]QueriedAsk, error) {
-	color.Blue(".. getting miner list")
+	isTTY := true
+	if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) == 0 {
+		isTTY = false
+	}
+	if isTTY {
+		color.Blue(".. getting miner list")
+	}
 	miners, err := api.StateListMiners(ctx, types.EmptyTSK)
 	if err != nil {
 		return nil, xerrors.Errorf("getting miner list: %w", err)
@@ -1282,14 +1298,18 @@ loop:
 	for {
 		select {
 		case <-time.After(150 * time.Millisecond):
-			fmt.Printf("\r* Found %d miners with power", atomic.LoadInt64(&found))
+			if isTTY {
+				fmt.Printf("\r* Found %d miners with power", atomic.LoadInt64(&found))
+			}
 		case <-done:
 			break loop
 		}
 	}
-	fmt.Printf("\r* Found %d miners with power\n", atomic.LoadInt64(&found))
+	if isTTY {
+		fmt.Printf("\r* Found %d miners with power\n", atomic.LoadInt64(&found))
 
-	color.Blue(".. querying asks")
+		color.Blue(".. querying asks")
+	}
 
 	var asks []QueriedAsk
 	var queried, got int64
@@ -1349,12 +1369,16 @@ loop2:
 	for {
 		select {
 		case <-time.After(150 * time.Millisecond):
-			fmt.Printf("\r* Queried %d asks, got %d responses", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
+			if isTTY {
+				fmt.Printf("\r* Queried %d asks, got %d responses", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
+			}
 		case <-done:
 			break loop2
 		}
 	}
-	fmt.Printf("\r* Queried %d asks, got %d responses\n", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
+	if isTTY {
+		fmt.Printf("\r* Queried %d asks, got %d responses\n", atomic.LoadInt64(&queried), atomic.LoadInt64(&got))
+	}
 
 	sort.Slice(asks, func(i, j int) bool {
 		return asks[i].Ask.Price.LessThan(asks[j].Ask.Price)

cli/cmd.go

@@ -1,34 +1,17 @@
 package cli
 
 import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
 	"strings"
-	"syscall"
 
 	logging "github.com/ipfs/go-log/v2"
-	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-jsonrpc"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/client"
 	cliutil "github.com/filecoin-project/lotus/cli/util"
-	"github.com/filecoin-project/lotus/node/repo"
 )
 
 var log = logging.Logger("cli")
 
-const (
-	metadataTraceContext = "traceContext"
-)
-
 // custom CLI error
 
 type ErrCmdFailed struct {
@@ -46,253 +29,31 @@ func NewCliError(s string) error {
 // ApiConnector returns API instance
 type ApiConnector func() api.FullNode
 
-// The flag passed on the command line with the listen address of the API
+func GetFullNodeServices(ctx *cli.Context) (ServicesAPI, error) {
-// server (only used by the tests)
+	if tn, ok := ctx.App.Metadata["test-services"]; ok {
-func flagForAPI(t repo.RepoType) string {
+		return tn.(ServicesAPI), nil
-	switch t {
-	case repo.FullNode:
-		return "api-url"
-	case repo.StorageMiner:
-		return "miner-api-url"
-	case repo.Worker:
-		return "worker-api-url"
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-func flagForRepo(t repo.RepoType) string {
-	switch t {
-	case repo.FullNode:
-		return "repo"
-	case repo.StorageMiner:
-		return "miner-repo"
-	case repo.Worker:
-		return "worker-repo"
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-func envForRepo(t repo.RepoType) string {
-	switch t {
-	case repo.FullNode:
-		return "FULLNODE_API_INFO"
-	case repo.StorageMiner:
-		return "MINER_API_INFO"
-	case repo.Worker:
-		return "WORKER_API_INFO"
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-// TODO remove after deprecation period
-func envForRepoDeprecation(t repo.RepoType) string {
-	switch t {
-	case repo.FullNode:
-		return "FULLNODE_API_INFO"
-	case repo.StorageMiner:
-		return "STORAGE_API_INFO"
-	case repo.Worker:
-		return "WORKER_API_INFO"
-	default:
-		panic(fmt.Sprintf("Unknown repo type: %v", t))
-	}
-}
-
-func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) {
-	// Check if there was a flag passed with the listen address of the API
-	// server (only used by the tests)
-	apiFlag := flagForAPI(t)
-	if ctx.IsSet(apiFlag) {
-		strma := ctx.String(apiFlag)
-		strma = strings.TrimSpace(strma)
-
-		return cliutil.APIInfo{Addr: strma}, nil
 	}
 
-	envKey := envForRepo(t)
+	api, c, err := GetFullNodeAPI(ctx)
-	env, ok := os.LookupEnv(envKey)
-	if !ok {
-		// TODO remove after deprecation period
-		envKey = envForRepoDeprecation(t)
-		env, ok = os.LookupEnv(envKey)
-		if ok {
-			log.Warnf("Use deprecation env(%s) value, please use env(%s) instead.", envKey, envForRepo(t))
-		}
-	}
-	if ok {
-		return cliutil.ParseApiInfo(env), nil
-	}
-
-	repoFlag := flagForRepo(t)
-
-	p, err := homedir.Expand(ctx.String(repoFlag))
 	if err != nil {
-		return cliutil.APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err)
+		return nil, err
 	}
 
-	r, err := repo.NewFS(p)
+	return &ServicesImpl{api: api, closer: c}, nil
-	if err != nil {
-		return cliutil.APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err)
-	}
-
-	ma, err := r.APIEndpoint()
-	if err != nil {
-		return cliutil.APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err)
-	}
-
-	token, err := r.APIToken()
-	if err != nil {
-		log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err)
-	}
-
-	return cliutil.APIInfo{
-		Addr:  ma.String(),
-		Token: token,
-	}, nil
 }
 
-func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) {
+var GetAPIInfo = cliutil.GetAPIInfo
-	ainfo, err := GetAPIInfo(ctx, t)
+var GetRawAPI = cliutil.GetRawAPI
-	if err != nil {
+var GetAPI = cliutil.GetAPI
-		return "", nil, xerrors.Errorf("could not get API info: %w", err)
-	}
 
-	addr, err := ainfo.DialArgs()
+var DaemonContext = cliutil.DaemonContext
-	if err != nil {
+var ReqContext = cliutil.ReqContext
-		return "", nil, xerrors.Errorf("could not get DialArgs: %w", err)
-	}
 
-	return addr, ainfo.AuthHeader(), nil
+var GetFullNodeAPI = cliutil.GetFullNodeAPI
-}
+var GetGatewayAPI = cliutil.GetGatewayAPI
 
-func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
+var GetStorageMinerAPI = cliutil.GetStorageMinerAPI
-	ti, ok := ctx.App.Metadata["repoType"]
+var GetWorkerAPI = cliutil.GetWorkerAPI
-	if !ok {
-		log.Errorf("unknown repo type, are you sure you want to use GetAPI?")
-		ti = repo.FullNode
-	}
-	t, ok := ti.(repo.RepoType)
-	if !ok {
-		log.Errorf("repoType type does not match the type of repo.RepoType")
-	}
-
-	if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
-		return tn.(api.StorageMiner), func() {}, nil
-	}
-	if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
-		return tn.(api.FullNode), func() {}, nil
-	}
-
-	addr, headers, err := GetRawAPI(ctx, t)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return client.NewCommonRPC(ctx.Context, addr, headers)
-}
-
-func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) {
-	if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
-		return tn.(api.FullNode), func() {}, nil
-	}
-
-	addr, headers, err := GetRawAPI(ctx, repo.FullNode)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return client.NewFullNodeRPC(ctx.Context, addr, headers)
-}
-
-type GetStorageMinerOptions struct {
-	PreferHttp bool
-}
-
-type GetStorageMinerOption func(*GetStorageMinerOptions)
-
-func StorageMinerUseHttp(opts *GetStorageMinerOptions) {
-	opts.PreferHttp = true
-}
-
-func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) {
-	var options GetStorageMinerOptions
-	for _, opt := range opts {
-		opt(&options)
-	}
-
-	if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
-		return tn.(api.StorageMiner), func() {}, nil
-	}
-
-	addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if options.PreferHttp {
-		u, err := url.Parse(addr)
-		if err != nil {
-			return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err)
-		}
-
-		switch u.Scheme {
-		case "ws":
-			u.Scheme = "http"
-		case "wss":
-			u.Scheme = "https"
-		}
-
-		addr = u.String()
-	}
-
-	return client.NewStorageMinerRPC(ctx.Context, addr, headers)
-}
-
-func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
-	addr, headers, err := GetRawAPI(ctx, repo.Worker)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return client.NewWorkerRPC(ctx.Context, addr, headers)
-}
-
-func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
-	addr, headers, err := GetRawAPI(ctx, repo.FullNode)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return client.NewGatewayRPC(ctx.Context, addr, headers)
-}
-
-func DaemonContext(cctx *cli.Context) context.Context {
-	if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok {
-		return mtCtx.(context.Context)
-	}
-
-	return context.Background()
-}
-
-// ReqContext returns context for cli execution. Calling it for the first time
-// installs SIGTERM handler that will close returned context.
-// Not safe for concurrent execution.
-func ReqContext(cctx *cli.Context) context.Context {
-	tCtx := DaemonContext(cctx)
-
-	ctx, done := context.WithCancel(tCtx)
-	sigChan := make(chan os.Signal, 2)
-	go func() {
-		<-sigChan
-		done()
-	}()
-	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
-
-	return ctx
-}
 
 var CommonCommands = []*cli.Command{
 	netCmd,

cli/disputer.go

@@ -20,10 +20,14 @@ import (
 	builtin3 "github.com/filecoin-project/specs-actors/v3/actors/builtin"
 	"golang.org/x/xerrors"
 
+	logging "github.com/ipfs/go-log/v2"
+
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/urfave/cli/v2"
 )
 
+var disputeLog = logging.Logger("disputer")
+
 const Confidence = 10
 
 type minerDeadline struct {
@@ -165,13 +169,13 @@ var disputerStartCmd = &cli.Command{
 			startEpoch = abi.ChainEpoch(cctx.Uint64("height"))
 		}
 
-		fmt.Println("checking sync status")
+		disputeLog.Info("checking sync status")
 
 		if err := SyncWait(ctx, api, false); err != nil {
 			return xerrors.Errorf("sync wait: %w", err)
 		}
 
-		fmt.Println("setting up window post disputer")
+		disputeLog.Info("setting up window post disputer")
 
 		// subscribe to head changes and validate the current value
 
@@ -220,10 +224,10 @@ var disputerStartCmd = &cli.Command{
 		statusCheckTicker := time.NewTicker(time.Hour)
 		defer statusCheckTicker.Stop()
 
-		fmt.Println("starting up window post disputer")
+		disputeLog.Info("starting up window post disputer")
 
 		applyTsk := func(tsk types.TipSetKey) error {
-			log.Infof("last checked height: %d", lastEpoch)
+			disputeLog.Infow("last checked epoch", "epoch", lastEpoch)
 			dls, ok := deadlineMap[lastEpoch]
 			delete(deadlineMap, lastEpoch)
 			if !ok || startEpoch >= lastEpoch {
@@ -261,12 +265,12 @@ var disputerStartCmd = &cli.Command{
 
 			// TODO: Parallelizeable / can be integrated into the previous deadline-iterating for loop
 			for _, dpmsg := range dpmsgs {
-				log.Infof("disputing a PoSt from miner %s", dpmsg.To)
+				disputeLog.Infow("disputing a PoSt", "miner", dpmsg.To)
 				m, err := api.MpoolPushMessage(ctx, dpmsg, mss)
 				if err != nil {
-					log.Infof("failed to dispute post message: %s", err.Error())
+					disputeLog.Errorw("failed to dispute post message", "err", err.Error(), "miner", dpmsg.To)
 				} else {
-					log.Infof("disputed a PoSt in message: %s", m.Cid())
+					disputeLog.Infow("submited dispute", "mcid", m.Cid(), "miner", dpmsg.To)
 				}
 			}
 
@@ -296,7 +300,7 @@ var disputerStartCmd = &cli.Command{
 					}
 				}
 			case <-statusCheckTicker.C:
-				log.Infof("Running status check: ")
+				disputeLog.Infof("running status check")
 
 				minerList, err = api.StateListMiners(ctx, types.EmptyTSK)
 				if err != nil {
@@ -321,14 +325,14 @@ var disputerStartCmd = &cli.Command{
 					// if an epoch got "skipped" from the deadlineMap somehow, just fry it now instead of letting it sit around forever
 					_, ok := deadlineMap[lastStatusCheckEpoch]
 					if ok {
-						log.Infof("epoch %d was skipped during execution, deleting it from deadlineMap")
+						disputeLog.Infow("epoch skipped during execution, deleting it from deadlineMap", "epoch", lastStatusCheckEpoch)
 						delete(deadlineMap, lastStatusCheckEpoch)
 					}
 				}
 
-				log.Infof("Status check complete")
+				log.Infof("status check complete")
 			case <-ctx.Done():
-				return xerrors.Errorf("context cancelled")
+				return ctx.Err()
 			}
 
 			return nil
@@ -336,10 +340,14 @@ var disputerStartCmd = &cli.Command{
 
 		for {
 			err := disputeLoop()
-			if err != nil {
+			if err == context.Canceled {
-				fmt.Println("disputer shutting down: ", err)
+				disputeLog.Info("disputer shutting down")
 				break
 			}
+			if err != nil {
+				disputeLog.Errorw("disputer shutting down", "err", err)
+				return err
+			}
 		}
 
 		return nil

cli/multisig.go

@@ -29,7 +29,7 @@ import (
 	init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init"
 	msig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig"
 
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
@@ -202,7 +202,7 @@ var msigInspectCmd = &cli.Command{
 		defer closer()
 		ctx := ReqContext(cctx)
 
-		store := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(api)))
+		store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(api)))
 
 		maddr, err := address.NewFromString(cctx.Args().First())
 		if err != nil {
@@ -1275,7 +1275,7 @@ var msigLockApproveCmd = &cli.Command{
 		params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{
 			StartEpoch:     abi.ChainEpoch(start),
 			UnlockDuration: abi.ChainEpoch(duration),
-			Amount:         abi.NewTokenAmount(amount.Int64()),
+			Amount:         big.Int(amount),
 		})
 
 		if actErr != nil {
@@ -1367,7 +1367,7 @@ var msigLockCancelCmd = &cli.Command{
 		params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{
 			StartEpoch:     abi.ChainEpoch(start),
 			UnlockDuration: abi.ChainEpoch(duration),
-			Amount:         abi.NewTokenAmount(amount.Int64()),
+			Amount:         big.Int(amount),
 		})
 
 		if actErr != nil {