diff --git a/.circleci/config.yml b/.circleci/config.yml index 00c565978..f863d6817 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1107,4 +1107,4 @@ workflows: only: - master jobs: - - publish-packer-snap + - publish-packer-snap \ No newline at end of file diff --git a/.circleci/template.yml b/.circleci/template.yml index d67f43e86..82e5bb8f6 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -912,4 +912,4 @@ workflows: only: - master jobs: - - publish-packer-snap + - publish-packer-snap \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 330a14d85..5eef6bb99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,129 @@ # Lotus changelog +# 1.15.1 / 2022-04-07 + +This is a *HIGHLY recommended* feature release v1.15.1, especially for node operators and storage providers who want to be a part of the content addressing network of Filecoin and IPFS. +This feature release introduces Index Provider, GraphSync v2, and many other latest functionalities, improvements and bug fixes. More importantly, node operator can now enable the FVM(experimental) to sync mainnet!! + +## Highlights + +### 🔥🔥🔥 FVM (Experimental) 🔥🔥🔥 +- feat: fvm: FVM integration ([filecoin-project/lotus#8332](https://github.com/filecoin-project/lotus/pull/8332)) +The lotus team is excited to announce the launch of experimental non-programmable FVM on mainnet. By enabling `"LOTUS_USE_FVM_EXPERIMENTAL=1` envvar, the lotus daemon will be running the [WASM-compiled built-in actors](https://github.com/filecoin-project/builtin-actors) that is compatible with the existing chain(Network v15 OhSnap). If you are trying it out and having any questions or feedbacks, please leave a comment [here](https://github.com/filecoin-project/lotus/discussions/8334)! + - chore: FVM: log when fvm is used([filecoin-project/lotus#8363](https://github.com/filecoin-project/lotus/pull/8363)) + - chore: ffi: the latest fvm release([filecoin-project/lotus#8382](https://github.com/filecoin-project/lotus/pull/8382)) + +### 🌟🌟🌟 Index Provider (Production Ready!) 🌟🌟🌟 +- feat: markets: Integrate index ingest protocol and retrieve by any CID ([filecoin-project/lotus#7313](https://github.com/filecoin-project/lotus/pull/7313)) + +More and more useful data is being stored on Filecoin via deals made by clients to Storage Providers. The goal is that this content is discoverable when people need them. To achieve that goal, one of the projects [the Bedrock team](https://www.notion.so/pl-strflt/Bedrock-2e956d5d8143432080a1d84435ccf0ff) is working on is building an Indexer Ecosystem, a project that's focus on content addressing on Filecoin, then potentially have interoperability with IPFS in the future and eventually serve the retrieval market. The Indexer Ecosystem high level architecture overview diagram can be found [here](https://github.com/filecoin-project/storetheindex/blob/main/doc/indexer_ecosys.png) and a detailed write up about can be found [here](https://www.notion.so/pl-strflt/Introducing-Indexer-to-SP-90bf296794174a8281c121d4ce6747a0). + +That being said, with this release, lotus Storage Providers can easily become an Index Provider and serve the Indexer Content Addressing System. Index Providers generate advertisements from the deals made by a storage provider and announces the data to the indexer nodes for further processing: +- To learn more about *what is an Index Provider and how to be an Index Provider*, read it [here](https://lotus.filecoin.io/storage-providers/operate/index-provider/) in lotus docuementation. +- An [one-off migration](https://lotus.filecoin.io/storage-providers/operate/index-provider/#first-time-migration) is needed in order for a Storage Provider to become an Index Provider and announce the proper formatted index. It's *highly recommended* for all Index Provider to do a [force bulk initialization](https://lotus.filecoin.io/storage-providers/configure/dagstore/#forcing-bulk-initialization) to enable index announcement on all existing deals. + - Note that the Initialization places IO workload on your storage system. SP should set a proper `concurrency` based on your hardware or can stop/start initialization at their wish/convenience as proving deadlines approach and elapse, to avoid IOPS starvation or competition with window PoSt. +- After the first one-time migration, being an Index Provider barely puts any extra usage on SP's market system. + - You can find the testing result by SPX fellows [here](https://github.com/filecoin-project/lotus/discussions/8087). + +We recommend all Storage Providers that are serving deals in the Filecoin network to become a Index Provider, make the data you are storing discoverable for the retrieval market and retrieval clients! + - If you have any questions about becoming an index provider, or the indexer system in general, leave a comment [here](https://github.com/filecoin-project/lotus/discussions/8341). + - Follow the indexer project at https://github.com/filecoin-project/go-indexer-core. + - If you have any feature request or bug reports of running an index provider, create an issue in https://github.com/filecoin-project/index-provider. + - You may also join the #storetheindex channel in the Filecoin Slack to engage with the team & the community! + +### ❗️❗️❗️ Dag Migration For New CAR index format in DagStore ❗️❗️❗️ +The index provider leverages the latest CARv2 indexing format `MultihashIndexSorted`, which stores the multihash code as well as the digest of all CIDs in a CAR file. Thus, all Storage Providers SHOULD perform an one-off DAG mirgation to regenerate DagStore CARv2 indices. You have to do it to become an index provider, failing to do so may also impact your future deal making. +Follow the instruction [here](https://lotus.filecoin.io/storage-providers/operate/index-provider/) to perform the migration. + +## New Features +- feat: sealing: Sector upgrade queue ([filecoin-project/lotus#8333](https://github.com/filecoin-project/lotus/pull/8333)) + - see more details in docs: [here](https://lotus.filecoin.io/storage-providers/operate/snap-deals/#snap-deal-queue) +- feat: market utils: Support unixfsnode in TraverseDag ([filecoin-project/lotus#8168](https://github.com/filecoin-project/lotus/pull/8168)) +- feat: config: enable indexer providing by default ([filecoin-project/lotus#8314](https://github.com/filecoin-project/lotus/pull/8314)) +- feat: api: Make ClientCalcCommP multithreaded ([filecoin-project/lotus#8276](https://github.com/filecoin-project/lotus/pull/8276)) +- feat: config: Persistent subsystem log level config ([filecoin-project/lotus#8283](https://github.com/filecoin-project/lotus/pull/8283)) +- feat: shed: blockstore/vlog to car export cmds ([filecoin-project/lotus#8265](https://github.com/filecoin-project/lotus/pull/8265)) +- feat: shed: ItestD ([filecoin-project/lotus#8290](https://github.com/filecoin-project/lotus/pull/8290)) +- feat: Make add piece idempotent ([filecoin-project/lotus#8160](https://github.com/filecoin-project/lotus/pull/8160)) +- feat: paychmgr: Support paych funding (a.k.a. fast paid retrieval) ([filecoin-project/lotus#7883](https://github.com/filecoin-project/lotus/pull/7883)) +- feat: ci: packer snap ([filecoin-project/lotus#7819](https://github.com/filecoin-project/lotus/pull/7819)) +- feat: #6147: Include worker name in sealing errors ([filecoin-project/lotus#7844](https://github.com/filecoin-project/lotus/pull/7844)) +- Feat: cli: Remove verified data cap ([filecoin-project/lotus#8175](https://github.com/filecoin-project/lotus/pull/8175)) +- feat: gateway: add MsigGetVestingSchedule to gateway api ([filecoin-project/lotus#8104](https://github.com/filecoin-project/lotus/pull/8104)) +- feat: itests: add itests ensemble mocknet getter ([filecoin-project/lotus#8157](https://github.com/filecoin-project/lotus/pull/8157)) +- feat: lotus-miner sectors list --initial-pledge ([filecoin-project/lotus#8098](https://github.com/filecoin-project/lotus/pull/8098)) +- Resource Manager Metrics ([filecoin-project/lotus#8089](https://github.com/filecoin-project/lotus/pull/8089)) +- feat: cli: set current network version from params ([filecoin-project/lotus#8111](https://github.com/filecoin-project/lotus/pull/8111)) +- feat: Snapdeals support in `storage find` CLI ([filecoin-project/lotus#8130](https://github.com/filecoin-project/lotus/pull/8130)) + +## Improvements +- improve resource manager integration ([filecoin-project/lotus#8318](https://github.com/filecoin-project/lotus/pull/8318)) +- add check manual-stateless-deal with interactive deal making ([filecoin-project/lotus#7560](https://github.com/filecoin-project/lotus/pull/7560)) +- test: cli: adding wallet tests ([filecoin-project/lotus#8079](https://github.com/filecoin-project/lotus/pull/8079)) +- test: chain: unit tests for the syncer & sync manager ([filecoin-project/lotus#8072](https://github.com/filecoin-project/lotus/pull/8072)) +- test: cli: unit tests for sync related commands ([filecoin-project/lotus#8080](https://github.com/filecoin-project/lotus/pull/8080)) +- misc: wallet: wallet tests with annotations for system test matrix ([filecoin-project/lotus#7928](https://github.com/filecoin-project/lotus/pull/7928)) +- test: Cli: add mempool tests ([filecoin-project/lotus#8162](https://github.com/filecoin-project/lotus/pull/8162)) +- add a state-tree diff command to lotus shed ([filecoin-project/lotus#8081](https://github.com/filecoin-project/lotus/pull/8081)) +- test: mempool: Add unit and integration tests ([filecoin-project/lotus#8017](https://github.com/filecoin-project/lotus/pull/8017)) +- splistore cold object reification redux ([filecoin-project/lotus#8029](https://github.com/filecoin-project/lotus/pull/8029)) +- test: cli: chain category unit tests ([filecoin-project/lotus#8048](https://github.com/filecoin-project/lotus/pull/8048)) +- feat: config: Move MakeNewSectorForDeals config into the Sealing section([filecoin-project/lotus#8382](https://github.com/filecoin-project/lotus/pull/8382)) + +## Bug Fixes +- fix: FVM: add finality check for consensus faults #8452 +- fix: market: Reuse the market PubSub in index provider #8451 +- fix: market: set all index provider options based on lotus config #8444 +- fix: sealing: Fix PR1 worker selection (#8421) +- fix: miner: dead loop on removing sector (#8421) +- fix: sealing: Remove sector copies from workers after snapdeals ([filecoin-project/lotus#8331](https://github.com/filecoin-project/lotus/pull/8331)) +- fix: storagefsm: Fix error loop on bad event ([filecoin-project/lotus#8339](https://github.com/filecoin-project/lotus/pull/8339)) +- fix: sealing: FinalizeSector doesn't need sealed replica access ([filecoin-project/lotus#8339](https://github.com/filecoin-project/lotus/pull/8339)) +- fix: sealing: always do cooldown in handleSubmitReplicaUpdateFailed ([filecoin-project/lotus#8353](https://github.com/filecoin-project/lotus/pull/8353)) +- fix: storage cli: Output primary sector status correctly ([filecoin-project/lotus#8320](https://github.com/filecoin-project/lotus/pull/8320)) +- fix: sealing fsm: Handle inputLk correctly ([filecoin-project/lotus#8291](https://github.com/filecoin-project/lotus/pull/8291)) +- fix: piece provider: Don't log CIDs as binary ([filecoin-project/lotus#8287](https://github.com/filecoin-project/lotus/pull/8287)) +- fix:sealing:Log instead of error normal shutdown of state machine ([filecoin-project/lotus#8232](https://github.com/filecoin-project/lotus/pull/8232)) +- fix:sealing:Handle finalize replica update failures in fsm ([filecoin-project/lotus#8229](https://github.com/filecoin-project/lotus/pull/8229)) +- ci: appimage: re-install appimage CI ([filecoin-project/lotus#7943](https://github.com/filecoin-project/lotus/pull/7943)) +- fix: sealing: PRU insufficient collateral ([filecoin-project/lotus#8219](https://github.com/filecoin-project/lotus/pull/8219)) +- fix: shed: diff command ([filecoin-project/lotus#8202](https://github.com/filecoin-project/lotus/pull/8202)) +- Make `--lite` option visible in the lotus daemon help text ([filecoin-project/lotus#8207](https://github.com/filecoin-project/lotus/pull/8207)) +- fix:sealing:Less verbose sector manager logging ([filecoin-project/lotus#8213](https://github.com/filecoin-project/lotus/pull/8213)) +- avoid panic ([filecoin-project/lotus#8205](https://github.com/filecoin-project/lotus/pull/8205)) +- A package is vulnerable to Exposure of Sensitive Information ([filecoin-project/lotus#8204](https://github.com/filecoin-project/lotus/pull/8204)) +- fix: sealing: add flag usage ([filecoin-project/lotus#8190](https://github.com/filecoin-project/lotus/pull/8190)) +- Fix the epoch used for gas in the message pool & validation ([filecoin-project/lotus#8163](https://github.com/filecoin-project/lotus/pull/8163)) +- fix:sealing:really-do-it flag for abort upgrade ([filecoin-project/lotus#8181](https://github.com/filecoin-project/lotus/pull/8181)) +- fix:proving:post check sector handles snap deals replica faults ([filecoin-project/lotus#8177](https://github.com/filecoin-project/lotus/pull/8177)) +- fix: client: calculate commps for pieces bigger than 32GB ([filecoin-project/lotus#8179](https://github.com/filecoin-project/lotus/pull/8179)) +- fix:cli:Continue instead of return error if no valid value is filled ([filecoin-project/lotus#8131](https://github.com/filecoin-project/lotus/pull/8131)) +- fix: limit reification sizes ([filecoin-project/lotus#8149](https://github.com/filecoin-project/lotus/pull/8149)) +- fix: state: Allow lotus-miner info to complete without admin permission ([filecoin-project/lotus#8057](https://github.com/filecoin-project/lotus/pull/8057)) +- fix:paychan:deflake integration test ([filecoin-project/lotus#8088](https://github.com/filecoin-project/lotus/pull/8088)) +- fix: worker: allow enable/disabling ReplicaUpdate tasks ([filecoin-project/lotus#8090](https://github.com/filecoin-project/lotus/pull/8090)) +- don't fail reification on missing references ([filecoin-project/lotus#8128](https://github.com/filecoin-project/lotus/pull/8128)) +- sealer: fix error message ([filecoin-project/lotus#8121](https://github.com/filecoin-project/lotus/pull/8121)) +- don't track peer ids in rcmgr metrics ([filecoin-project/lotus#8099](https://github.com/filecoin-project/lotus/pull/8099)) +- temporarily disable reification ([filecoin-project/lotus#8132](https://github.com/filecoin-project/lotus/pull/8132)) +- [Describe]: when excute cmd "lotus-bench sealing" without "benchmark-… ([filecoin-project/lotus#8173](https://github.com/filecoin-project/lotus/pull/8173)) + +## Dependency Updates +- deps: update go-libp2p and go-libp2p-resource-manager ([filecoin-project/lotus#8289](https://github.com/filecoin-project/lotus/pull/8289)) +- feat(deps): update to graphsync v0.13.0 with 2.0 protocol ([filecoin-project/lotus#8273](https://github.com/filecoin-project/lotus/pull/8273)) +- dep: actor: get v7 ([filecoin-project/lotus#8194](https://github.com/filecoin-project/lotus/pull/8194)) +- deps: update go-libp2p to v0.18 release ([filecoin-project/lotus#8355](https://github.com/filecoin-project/lotus/pull/8355)) +- github.com/filecoin-project/go-data-transfer (v1.14.1 -> v1.15.0): +- github.com/filecoin-project/go-fil-markets (v1.19.2 -> v1.20.1): +- deps: update go-libp2p to v0.18.0-rc5 ([filecoin-project/lotus#8169](https://github.com/filecoin-project/lotus/pull/8169)) + +## Others +- chore: build: backport releases ([filecoin-project/lotus#8192](https://github.com/filecoin-project/lotus/pull/8192)) +- feat: build: bump the version to v1.15.1-dev ([filecoin-project/lotus#8073](https://github.com/filecoin-project/lotus/pull/8073)) +- makefile: add make jen ([filecoin-project/lotus#8122](https://github.com/filecoin-project/lotus/pull/8122)) +- chore: Merge releases into master ([filecoin-project/lotus#8156](https://github.com/filecoin-project/lotus/pull/8156)) +- chore: ci: disable publish-packer #8451 + # 1.15.0 / 2022-03-09 This is an optional release with retrieval improvements(client side), SP ux with unsealing, snap deals and regular deal making and many other new features, improvements and bug fixes. diff --git a/Dockerfile.lotus b/Dockerfile.lotus index 4609ff9a2..4a7001178 100644 --- a/Dockerfile.lotus +++ b/Dockerfile.lotus @@ -253,3 +253,4 @@ EXPOSE 1234 EXPOSE 2345 EXPOSE 3456 EXPOSE 1777 + diff --git a/chain/vm/fvm.go b/chain/vm/fvm.go index daa4d3c02..f48c5bdbd 100644 --- a/chain/vm/fvm.go +++ b/chain/vm/fvm.go @@ -5,6 +5,9 @@ import ( "context" "time" + cid "github.com/ipfs/go-cid/_rsrch/cidiface" + + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/go-state-types/network" @@ -15,7 +18,6 @@ import ( "github.com/filecoin-project/lotus/chain/state" cbor "github.com/ipfs/go-ipld-cbor" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/lotus/lib/sigs" @@ -30,7 +32,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" ) var _ Interface = (*FVM)(nil) diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 81deed045..03eccd29a 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -7,7 +7,11 @@ USAGE: lotus-miner [global options] command [command options] [arguments...] VERSION: +<<<<<<< HEAD 1.15.2-dev +======= + 1.15.1 +>>>>>>> releases COMMANDS: init Initialize a lotus miner repo diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index 52445093a..285ca4505 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -7,7 +7,11 @@ USAGE: lotus-worker [global options] command [command options] [arguments...] VERSION: +<<<<<<< HEAD 1.15.2-dev +======= + 1.15.1 +>>>>>>> releases COMMANDS: run Start lotus worker diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index a9c729930..fbb7b6598 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -7,7 +7,11 @@ USAGE: lotus [global options] command [command options] [arguments...] VERSION: +<<<<<<< HEAD 1.15.2-dev +======= + 1.15.1 +>>>>>>> releases COMMANDS: daemon Start a lotus daemon process diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index 4ddac8e0b..22c2b1ce8 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -269,7 +269,7 @@ # # type: bool # env var: LOTUS_INDEXPROVIDER_ENABLE - #Enable = false + #Enable = true # EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement # entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The diff --git a/export b/export new file mode 100644 index 000000000..e69de29bb diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index cc5786e7d..b3ed12040 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -7,16 +7,17 @@ import ( "net/http" "sync" - "github.com/google/uuid" + "github.com/filecoin-project/go-ds-versioning/pkg/statestore" "github.com/hashicorp/go-multierror" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" + + "github.com/google/uuid" + cid "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" "go.uber.org/multierr" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" diff --git a/extern/sector-storage/stores/index.go b/extern/sector-storage/stores/index.go index 5a0d0f3a3..d5039d9e8 100644 --- a/extern/sector-storage/stores/index.go +++ b/extern/sector-storage/stores/index.go @@ -7,6 +7,7 @@ import ( "net/url" gopath "path" "sort" + "strings" "sync" "time" @@ -25,6 +26,62 @@ import ( var HeartbeatInterval = 10 * time.Second var SkippedHeartbeatThresh = HeartbeatInterval * 5 +// ID identifies sector storage by UUID. One sector storage should map to one +// filesystem, local or networked / shared by multiple machines +type ID string + +const IDSep = "." + +type IDList []ID + +func (il IDList) String() string { + l := make([]string, len(il)) + for i, id := range il { + l[i] = string(id) + } + return strings.Join(l, IDSep) +} + +func ParseIDList(s string) IDList { + strs := strings.Split(s, IDSep) + out := make([]ID, len(strs)) + for i, str := range strs { + out[i] = ID(str) + } + return out +} + +type Group = string + +type StorageInfo struct { + ID ID + URLs []string // TODO: Support non-http transports + Weight uint64 + MaxStorage uint64 + + CanSeal bool + CanStore bool + + Groups []Group + AllowTo []Group +} + +type HealthReport struct { + Stat fsutil.FsStat + Err string +} + +type SectorStorageInfo struct { + ID ID + URLs []string // TODO: Support non-http transports + Weight uint64 + + CanSeal bool + CanStore bool + + Primary bool +} + //go:generate go run github.com/golang/mock/mockgen -destination=mocks/index.go -package=mocks . SectorIndex type SectorIndex interface { // part of storage-miner api diff --git a/extern/storage-sealing/input.go b/extern/storage-sealing/input.go index de3ae9f2a..5c03753a2 100644 --- a/extern/storage-sealing/input.go +++ b/extern/storage-sealing/input.go @@ -5,15 +5,13 @@ import ( "sort" "time" + cid "github.com/ipfs/go-cid/_rsrch/cidiface" + "golang.org/x/xerrors" - "github.com/ipfs/go-cid" - "github.com/filecoin-project/go-commp-utils/zerocomm" - "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index f2566a8c9..9d8421a33 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -6,14 +6,12 @@ import ( "sync" "time" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" + cid "github.com/ipfs/go-cid/_rsrch/cidiface" + "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" @@ -107,6 +105,8 @@ type Sealing struct { assignedPieces map[abi.SectorID][]cid.Cid nextDealSector *abi.SectorNumber // used to prevent a race where we could create a new sector more than once + upgradeLk sync.Mutex + toUpgrade map[abi.SectorNumber]struct{} available map[abi.SectorID]struct{} notifee SectorStateNotifee @@ -180,6 +180,8 @@ func New(mctx context.Context, api SealingAPI, fc config.MinerFeeConfig, events available: map[abi.SectorID]struct{}{}, + available: map[abi.SectorID]struct{}{}, + notifee: notifee, addrSel: as, diff --git a/extern/storage-sealing/states_proving.go b/extern/storage-sealing/states_proving.go index a5e5ddf88..d5881e744 100644 --- a/extern/storage-sealing/states_proving.go +++ b/extern/storage-sealing/states_proving.go @@ -6,7 +6,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/go-statemachine" + statemachine "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/policy" ) @@ -135,6 +135,21 @@ func (m *Sealing) handleProvingSector(ctx statemachine.Context, sector SectorInf delete(m.available, m.minerSectorID(sector.SectorNumber)) m.inputLk.Unlock() + cfg, err := m.getConfig() + if err != nil { + return xerrors.Errorf("getting sealing config: %w", err) + } + + // TODO: Watch termination + // TODO: Auto-extend if set + + return nil +} + +func (m *Sealing) handleAvailableSector(ctx statemachine.Context, sector SectorInfo) error { + m.inputLk.Lock() + m.available[m.minerSectorID(sector.SectorNumber)] = struct{}{} + m.inputLk.Unlock() // TODO: Watch termination // TODO: Auto-extend if set diff --git a/extern/storage-sealing/upgrade_queue.go b/extern/storage-sealing/upgrade_queue.go index 1ee228ccb..9bd3d5c60 100644 --- a/extern/storage-sealing/upgrade_queue.go +++ b/extern/storage-sealing/upgrade_queue.go @@ -6,6 +6,9 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" market7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/market" ) @@ -56,3 +59,60 @@ func (m *Sealing) sectorActive(ctx context.Context, tok TipSetToken, sector abi. return active.IsSet(uint64(sector)) } + +func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreCommitInfo) big.Int { + if len(params.DealIDs) == 0 { + return big.Zero() + } + replace := m.maybeUpgradableSector() + if replace != nil { + loc, err := m.Api.StateSectorPartition(ctx, m.maddr, *replace, nil) + if err != nil { + log.Errorf("error calling StateSectorPartition for replaced sector: %+v", err) + return big.Zero() + } + + params.ReplaceCapacity = true + params.ReplaceSectorNumber = *replace + params.ReplaceSectorDeadline = loc.Deadline + params.ReplaceSectorPartition = loc.Partition + + log.Infof("replacing sector %d with %d", *replace, params.SectorNumber) + + ri, err := m.Api.StateSectorGetInfo(ctx, m.maddr, *replace, nil) + if err != nil { + log.Errorf("error calling StateSectorGetInfo for replaced sector: %+v", err) + return big.Zero() + } + if ri == nil { + log.Errorf("couldn't find sector info for sector to replace: %+v", replace) + return big.Zero() + } + + if params.Expiration < ri.Expiration { + // TODO: Some limit on this + params.Expiration = ri.Expiration + } + + return ri.InitialPledge + } + + return big.Zero() +} + +func (m *Sealing) maybeUpgradableSector() *abi.SectorNumber { + m.upgradeLk.Lock() + defer m.upgradeLk.Unlock() + for number := range m.toUpgrade { + // TODO: checks to match actor constraints + + // this one looks good + /*if checks */ + { + delete(m.toUpgrade, number) + return &number + } + } + + return nil +} diff --git a/go.mod b/go.mod index 25831473a..5155c6c12 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/filecoin-project/go-commp-utils v0.1.3 github.com/filecoin-project/go-crypto v0.0.1 github.com/filecoin-project/go-data-transfer v1.15.1 + github.com/filecoin-project/go-ds-versioning v0.1.1 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 github.com/filecoin-project/go-fil-markets v1.20.1 @@ -105,7 +106,7 @@ require ( github.com/ipfs/interface-go-ipfs-core v0.5.2 github.com/ipld/go-car v0.3.3 github.com/ipld/go-car/v2 v2.1.1 - github.com/ipld/go-codec-dagpb v1.3.1 + github.com/ipld/go-codec-dagpb v1.3.2 github.com/ipld/go-ipld-prime v0.16.0 github.com/ipld/go-ipld-selector-text-lite v0.0.1 github.com/jonboulle/clockwork v0.2.2 // indirect diff --git a/go.sum b/go.sum index 17ebde021..086810b9d 100644 --- a/go.sum +++ b/go.sum @@ -913,8 +913,9 @@ github.com/ipld/go-car/v2 v2.1.1 h1:saaKz4nC0AdfCGHLYKeXLGn8ivoPC54fyS55uyOLKwA= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= -github.com/ipld/go-codec-dagpb v1.3.1 h1:yVNlWRQexCa54ln3MSIiUN++ItH7pdhBFhh0hSgZu1w= github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= +github.com/ipld/go-codec-dagpb v1.3.2 h1:MZQUIjanHXXfDuYmtWYT8nFbqfFsZuyHClj6VDmSXr4= +github.com/ipld/go-codec-dagpb v1.3.2/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= diff --git a/itests/ccupgrade_test.go b/itests/ccupgrade_test.go index cd876a3fd..71eb9b704 100644 --- a/itests/ccupgrade_test.go +++ b/itests/ccupgrade_test.go @@ -7,13 +7,17 @@ import ( "testing" "time" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/itests/kit" ) @@ -101,3 +105,104 @@ func runTestCCUpgrade(t *testing.T) *kit.TestFullNode { return client } + +func waitForSectorActive(ctx context.Context, t *testing.T, sn abi.SectorNumber, node *kit.TestFullNode, maddr address.Address) { + for { + active, err := node.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + for _, si := range active { + if si.SectorNumber == sn { + fmt.Printf("ACTIVE\n") + return + } + } + + time.Sleep(time.Second) + } +} + +func TestCCUpgradeAndPoSt(t *testing.T) { + kit.QuietMiningLogs() + t.Run("upgrade and then post", func(t *testing.T) { + ctx := context.Background() + n := runTestCCUpgrade(t) + ts, err := n.ChainHead(ctx) + require.NoError(t, err) + start := ts.Height() + // wait for a full proving period + t.Log("waiting for chain") + + n.WaitTillChain(ctx, func(ts *types.TipSet) bool { + if ts.Height() > start+abi.ChainEpoch(2880) { + return true + } + return false + }) + }) +} + +func TestAbortUpgradeAvailable(t *testing.T) { + kit.QuietMiningLogs() + + ctx := context.Background() + blockTime := 1 * time.Millisecond + + client, miner, ens := kit.EnsembleMinimal(t, kit.GenesisNetworkVersion(network.Version15), kit.ThroughRPC()) + ens.InterconnectAll().BeginMiningMustPost(blockTime) + + maddr, err := miner.ActorAddress(ctx) + if err != nil { + t.Fatal(err) + } + + CCUpgrade := abi.SectorNumber(kit.DefaultPresealsPerBootstrapMiner + 1) + fmt.Printf("CCUpgrade: %d\n", CCUpgrade) + + miner.PledgeSectors(ctx, 1, 0, nil) + sl, err := miner.SectorsList(ctx) + require.NoError(t, err) + require.Len(t, sl, 1, "expected 1 sector") + require.Equal(t, CCUpgrade, sl[0], "unexpected sector number") + { + si, err := client.StateSectorGetInfo(ctx, maddr, CCUpgrade, types.EmptyTSK) + require.NoError(t, err) + require.Less(t, 50000, int(si.Expiration)) + } + waitForSectorActive(ctx, t, CCUpgrade, client, maddr) + + err = miner.SectorMarkForUpgrade(ctx, sl[0], true) + require.NoError(t, err) + + sl, err = miner.SectorsList(ctx) + require.NoError(t, err) + require.Len(t, sl, 1, "expected 1 sector") + + ss, err := miner.SectorsStatus(ctx, sl[0], false) + require.NoError(t, err) + + for i := 0; i < 100; i++ { + ss, err = miner.SectorsStatus(ctx, sl[0], false) + require.NoError(t, err) + if ss.State == api.SectorState(sealing.Proving) { + time.Sleep(50 * time.Millisecond) + continue + } + + require.Equal(t, api.SectorState(sealing.Available), ss.State) + break + } + + require.NoError(t, miner.SectorAbortUpgrade(ctx, sl[0])) + + for i := 0; i < 100; i++ { + ss, err = miner.SectorsStatus(ctx, sl[0], false) + require.NoError(t, err) + if ss.State == api.SectorState(sealing.Available) { + time.Sleep(50 * time.Millisecond) + continue + } + + require.Equal(t, api.SectorState(sealing.Proving), ss.State) + break + } +} diff --git a/node/config/def.go b/node/config/def.go index 10e29c176..b7ab39d3d 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -193,7 +193,7 @@ func DefaultStorageMiner() *StorageMiner { }, IndexProvider: IndexProviderConfig{ - Enable: false, + Enable: true, EntriesCacheCapacity: 1024, EntriesChunkSize: 16384, TopicName: "/indexer/ingest/mainnet", diff --git a/node/config/def_test.go b/node/config/def_test.go index cc37a782e..9a450e66b 100644 --- a/node/config/def_test.go +++ b/node/config/def_test.go @@ -76,6 +76,6 @@ func TestDefaultMinerRoundtrip(t *testing.T) { func TestDefaultStorageMiner_SetsIndexIngestTopic(t *testing.T) { subject := DefaultStorageMiner() - require.False(t, subject.IndexProvider.Enable) + require.True(t, subject.IndexProvider.Enable) require.Equal(t, "/indexer/ingest/mainnet", subject.IndexProvider.TopicName) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index ba3fd57e3..93219ba10 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -5,17 +5,17 @@ import ( "context" "errors" "fmt" - "net/http" - "os" - "path/filepath" "strings" "time" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-storedcounter" + "github.com/ipfs/go-datastore" + "go.uber.org/fx" "go.uber.org/multierr" "golang.org/x/xerrors" - "github.com/filecoin-project/go-address" dtimpl "github.com/filecoin-project/go-data-transfer/impl" dtnet "github.com/filecoin-project/go-data-transfer/network" dtgstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync" @@ -30,14 +30,9 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-jsonrpc/auth" - "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/go-storedcounter" provider "github.com/filecoin-project/index-provider" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" graphsync "github.com/ipfs/go-graphsync/impl" gsnet "github.com/ipfs/go-graphsync/network" @@ -976,39 +971,37 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error func ToSealingConfig(dealmakingCfg config.DealmakingConfig, sealingCfg config.SealingConfig) sealiface.Config { return sealiface.Config{ - MaxWaitDealsSectors: sealingCfg.MaxWaitDealsSectors, - MaxSealingSectors: sealingCfg.MaxSealingSectors, - MaxSealingSectorsForDeals: sealingCfg.MaxSealingSectorsForDeals, - PreferNewSectorsForDeals: sealingCfg.PreferNewSectorsForDeals, - MaxUpgradingSectors: sealingCfg.MaxUpgradingSectors, - StartEpochSealingBuffer: abi.ChainEpoch(dealmakingCfg.StartEpochSealingBuffer), - MakeNewSectorForDeals: sealingCfg.MakeNewSectorForDeals, - CommittedCapacitySectorLifetime: time.Duration(sealingCfg.CommittedCapacitySectorLifetime), - WaitDealsDelay: time.Duration(sealingCfg.WaitDealsDelay), - MakeCCSectorsAvailable: sealingCfg.MakeCCSectorsAvailable, - AlwaysKeepUnsealedCopy: sealingCfg.AlwaysKeepUnsealedCopy, - FinalizeEarly: sealingCfg.FinalizeEarly, + MaxWaitDealsSectors: cfg.Sealing.MaxWaitDealsSectors, + MaxSealingSectors: cfg.Sealing.MaxSealingSectors, + MaxSealingSectorsForDeals: cfg.Sealing.MaxSealingSectorsForDeals, + StartEpochSealingBuffer: abi.ChainEpoch(cfg.Dealmaking.StartEpochSealingBuffer), + MakeNewSectorForDeals: cfg.Sealing.MakeNewSectorForDeals, + CommittedCapacitySectorLifetime: time.Duration(cfg.Sealing.CommittedCapacitySectorLifetime), + WaitDealsDelay: time.Duration(cfg.Sealing.WaitDealsDelay), + MakeCCSectorsAvailable: cfg.Sealing.MakeCCSectorsAvailable, + AlwaysKeepUnsealedCopy: cfg.Sealing.AlwaysKeepUnsealedCopy, + FinalizeEarly: cfg.Sealing.FinalizeEarly, - CollateralFromMinerBalance: sealingCfg.CollateralFromMinerBalance, - AvailableBalanceBuffer: types.BigInt(sealingCfg.AvailableBalanceBuffer), - DisableCollateralFallback: sealingCfg.DisableCollateralFallback, + CollateralFromMinerBalance: cfg.Sealing.CollateralFromMinerBalance, + AvailableBalanceBuffer: types.BigInt(cfg.Sealing.AvailableBalanceBuffer), + DisableCollateralFallback: cfg.Sealing.DisableCollateralFallback, - BatchPreCommits: sealingCfg.BatchPreCommits, - MaxPreCommitBatch: sealingCfg.MaxPreCommitBatch, - PreCommitBatchWait: time.Duration(sealingCfg.PreCommitBatchWait), - PreCommitBatchSlack: time.Duration(sealingCfg.PreCommitBatchSlack), + BatchPreCommits: cfg.Sealing.BatchPreCommits, + MaxPreCommitBatch: cfg.Sealing.MaxPreCommitBatch, + PreCommitBatchWait: time.Duration(cfg.Sealing.PreCommitBatchWait), + PreCommitBatchSlack: time.Duration(cfg.Sealing.PreCommitBatchSlack), - AggregateCommits: sealingCfg.AggregateCommits, - MinCommitBatch: sealingCfg.MinCommitBatch, - MaxCommitBatch: sealingCfg.MaxCommitBatch, - CommitBatchWait: time.Duration(sealingCfg.CommitBatchWait), - CommitBatchSlack: time.Duration(sealingCfg.CommitBatchSlack), - AggregateAboveBaseFee: types.BigInt(sealingCfg.AggregateAboveBaseFee), - BatchPreCommitAboveBaseFee: types.BigInt(sealingCfg.BatchPreCommitAboveBaseFee), + AggregateCommits: cfg.Sealing.AggregateCommits, + MinCommitBatch: cfg.Sealing.MinCommitBatch, + MaxCommitBatch: cfg.Sealing.MaxCommitBatch, + CommitBatchWait: time.Duration(cfg.Sealing.CommitBatchWait), + CommitBatchSlack: time.Duration(cfg.Sealing.CommitBatchSlack), + AggregateAboveBaseFee: types.BigInt(cfg.Sealing.AggregateAboveBaseFee), + BatchPreCommitAboveBaseFee: types.BigInt(cfg.Sealing.BatchPreCommitAboveBaseFee), - TerminateBatchMax: sealingCfg.TerminateBatchMax, - TerminateBatchMin: sealingCfg.TerminateBatchMin, - TerminateBatchWait: time.Duration(sealingCfg.TerminateBatchWait), + TerminateBatchMax: cfg.Sealing.TerminateBatchMax, + TerminateBatchMin: cfg.Sealing.TerminateBatchMin, + TerminateBatchWait: time.Duration(cfg.Sealing.TerminateBatchWait), } } diff --git a/testplans/lotus-soup/go.sum b/testplans/lotus-soup/go.sum index 48221e49d..4ad4101fa 100644 --- a/testplans/lotus-soup/go.sum +++ b/testplans/lotus-soup/go.sum @@ -1133,8 +1133,9 @@ github.com/ipld/go-car/v2 v2.1.1 h1:saaKz4nC0AdfCGHLYKeXLGn8ivoPC54fyS55uyOLKwA= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= -github.com/ipld/go-codec-dagpb v1.3.1 h1:yVNlWRQexCa54ln3MSIiUN++ItH7pdhBFhh0hSgZu1w= github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= +github.com/ipld/go-codec-dagpb v1.3.2 h1:MZQUIjanHXXfDuYmtWYT8nFbqfFsZuyHClj6VDmSXr4= +github.com/ipld/go-codec-dagpb v1.3.2/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=