2021-07-23 11:55:50 +00:00
// Code generated by github.com/filecoin-project/lotus/node/config/cfgdocgen. DO NOT EDIT.
package config
type DocField struct {
2022-09-14 16:41:47 +00:00
Name string
2021-07-23 11:55:50 +00:00
Type string
Comment string
}
var Doc = map [ string ] [ ] DocField {
2023-09-22 21:15:18 +00:00
"API" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "ListenAddress" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 12:55:19 +00:00
Comment : ` Binding address for the Lotus API ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "RemoteListenAddress" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Timeout" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
2023-10-25 19:13:56 +00:00
"ApisConfig" : {
{
2023-11-13 23:59:34 +00:00
Name : "ChainApiInfo" ,
2023-10-25 19:13:56 +00:00
Type : "[]string" ,
2023-11-13 23:59:34 +00:00
Comment : ` ChainApiInfo is the API endpoint for the Lotus daemon. ` ,
2023-10-25 19:13:56 +00:00
} ,
2023-11-06 22:10:57 +00:00
{
Name : "StorageRPCSecret" ,
Type : "string" ,
Comment : ` RPC Secret for the storage subsystem .
If integrating with lotus - miner this must match the value from
cat ~ / . lotusminer / keystore / MF2XI2BNNJ3XILLQOJUXMYLUMU | jq - r . PrivateKey ` ,
} ,
2023-10-25 19:13:56 +00:00
} ,
2023-09-22 21:15:18 +00:00
"Backup" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "DisableMetadataLog" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2022-07-01 20:21:19 +00:00
Comment : ` When set to true disables metadata log ( . lotus / kvlog ) . This can save disk
space by reducing metadata redundancy .
Note that in case of metadata corruption it might be much harder to recover
2021-07-23 13:40:30 +00:00
your node if metadata log is disabled ` ,
2021-07-23 11:55:50 +00:00
} ,
} ,
2023-09-22 21:15:18 +00:00
"BatchFeeConfig" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Base" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "PerSector" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"Chainstore" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "EnableSplitstore" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Splitstore" ,
2021-07-23 13:18:20 +00:00
Type : "Splitstore" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"Client" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "UseIpfs" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "IpfsOnlineMode" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "IpfsMAddr" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "IpfsUseForRetrieval" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "SimultaneousTransfersForStorage" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2021-07-23 13:40:30 +00:00
Comment : ` The maximum number of simultaneous data transfers between the client
2021-09-30 01:49:59 +00:00
and storage providers for storage deals ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "SimultaneousTransfersForRetrieval" ,
2021-09-30 01:49:59 +00:00
Type : "uint64" ,
Comment : ` The maximum number of simultaneous data transfers between the client
and storage providers for retrieval deals ` ,
2021-07-23 11:55:50 +00:00
} ,
2022-01-06 15:26:25 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "OffChainRetrieval" ,
2022-01-06 15:26:25 +00:00
Type : "bool" ,
2022-02-14 20:16:41 +00:00
Comment : ` Require that retrievals perform no on - chain operations . Paid retrievals
2022-01-06 15:26:25 +00:00
without existing payment channels with available funds will fail instead
of automatically performing on - chain operations . ` ,
} ,
2021-07-23 11:55:50 +00:00
} ,
2023-09-22 21:15:18 +00:00
"Common" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "API" ,
2021-07-23 13:18:20 +00:00
Type : "API" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Backup" ,
2021-07-23 13:18:20 +00:00
Type : "Backup" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
2022-03-10 10:58:31 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Logging" ,
2022-03-10 10:58:31 +00:00
Type : "Logging" ,
Comment : ` ` ,
} ,
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Libp2p" ,
2021-07-23 13:18:20 +00:00
Type : "Libp2p" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Pubsub" ,
2021-07-23 13:18:20 +00:00
Type : "Pubsub" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
2024-03-15 21:38:13 +00:00
"CurioAddresses" : {
{
Name : "PreCommitControl" ,
Type : "[]string" ,
Comment : ` Addresses to send PreCommit messages from ` ,
} ,
{
Name : "CommitControl" ,
Type : "[]string" ,
Comment : ` Addresses to send Commit messages from ` ,
} ,
{
Name : "TerminateControl" ,
Type : "[]string" ,
Comment : ` ` ,
} ,
{
Name : "DisableOwnerFallback" ,
Type : "bool" ,
Comment : ` DisableOwnerFallback disables usage of the owner address for messages
sent automatically ` ,
} ,
{
Name : "DisableWorkerFallback" ,
Type : "bool" ,
Comment : ` DisableWorkerFallback disables usage of the worker address for messages
sent automatically , if control addresses are configured .
A control address that doesn ' t have enough funds will still be chosen
over the worker address if this flag is set . ` ,
} ,
{
Name : "MinerAddresses" ,
Type : "[]string" ,
Comment : ` MinerAddresses are the addresses of the miner actors to use for sending messages ` ,
} ,
} ,
"CurioConfig" : {
{
Name : "Subsystems" ,
Type : "CurioSubsystemsConfig" ,
Comment : ` ` ,
} ,
{
Name : "Fees" ,
Type : "CurioFees" ,
Comment : ` ` ,
} ,
{
Name : "Addresses" ,
Type : "[]CurioAddresses" ,
Comment : ` Addresses of wallets per MinerAddress (one of the fields). ` ,
} ,
{
Name : "Proving" ,
2024-04-02 16:21:21 +00:00
Type : "CurioProvingConfig" ,
2024-03-15 21:38:13 +00:00
Comment : ` ` ,
} ,
{
Name : "Journal" ,
Type : "JournalConfig" ,
Comment : ` ` ,
} ,
{
Name : "Apis" ,
Type : "ApisConfig" ,
Comment : ` ` ,
} ,
} ,
"CurioFees" : {
{
Name : "DefaultMaxFee" ,
Type : "types.FIL" ,
Comment : ` ` ,
} ,
{
Name : "MaxPreCommitGasFee" ,
Type : "types.FIL" ,
Comment : ` ` ,
} ,
{
Name : "MaxCommitGasFee" ,
Type : "types.FIL" ,
Comment : ` ` ,
} ,
{
Name : "MaxPreCommitBatchGasFee" ,
Type : "BatchFeeConfig" ,
Comment : ` maxBatchFee = maxBase + maxPerSector * nSectors ` ,
} ,
{
Name : "MaxCommitBatchGasFee" ,
Type : "BatchFeeConfig" ,
Comment : ` ` ,
} ,
{
Name : "MaxTerminateGasFee" ,
Type : "types.FIL" ,
Comment : ` ` ,
} ,
{
Name : "MaxWindowPoStGasFee" ,
Type : "types.FIL" ,
Comment : ` WindowPoSt is a high-value operation, so the default fee should be high. ` ,
} ,
{
Name : "MaxPublishDealsFee" ,
Type : "types.FIL" ,
Comment : ` ` ,
} ,
} ,
2024-04-02 16:21:21 +00:00
"CurioProvingConfig" : {
{
Name : "ParallelCheckLimit" ,
Type : "int" ,
Comment : ` Maximum number of sector checks to run in parallel . ( 0 = unlimited )
WARNING : Setting this value too high may make the node crash by running out of stack
WARNING : Setting this value too low may make sector challenge reading much slower , resulting in failed PoSt due
to late submission .
After changing this option , confirm that the new value works in your setup by invoking
' lotus - miner proving compute window - post 0 ' ` ,
} ,
{
Name : "SingleCheckTimeout" ,
Type : "Duration" ,
Comment : ` Maximum amount of time a proving pre - check can take for a sector . If the check times out the sector will be skipped
WARNING : Setting this value too low risks in sectors being skipped even though they are accessible , just reading the
test challenge took longer than this timeout
WARNING : Setting this value too high risks missing PoSt deadline in case IO operations related to this sector are
blocked ( e . g . in case of disconnected NFS mount ) ` ,
} ,
{
Name : "PartitionCheckTimeout" ,
Type : "Duration" ,
Comment : ` Maximum amount of time a proving pre - check can take for an entire partition . If the check times out , sectors in
the partition which didn ' t get checked on time will be skipped
WARNING : Setting this value too low risks in sectors being skipped even though they are accessible , just reading the
test challenge took longer than this timeout
WARNING : Setting this value too high risks missing PoSt deadline in case IO operations related to this partition are
blocked or slow ` ,
} ,
{
Name : "DisableWDPoStPreChecks" ,
Type : "bool" ,
Comment : ` Disable WindowPoSt provable sector readability checks .
In normal operation , when preparing to compute WindowPoSt , lotus - miner will perform a round of reading challenges
from all sectors to confirm that those sectors can be proven . Challenges read in this process are discarded , as
we ' re only interested in checking that sector data can be read .
When using builtin proof computation ( no PoSt workers , and DisableBuiltinWindowPoSt is set to false ) , this process
can save a lot of time and compute resources in the case that some sectors are not readable - this is caused by
the builtin logic not skipping snark computation when some sectors need to be skipped .
When using PoSt workers , this process is mostly redundant , with PoSt workers challenges will be read once , and
if challenges for some sectors aren ' t readable , those sectors will just get skipped .
Disabling sector pre - checks will slightly reduce IO load when proving sectors , possibly resulting in shorter
time to produce window PoSt . In setups with good IO capabilities the effect of this option on proving time should
be negligible .
NOTE : It likely is a bad idea to disable sector pre - checks in setups with no PoSt workers .
NOTE : Even when this option is enabled , recovering sectors will be checked before recovery declaration message is
sent to the chain
After changing this option , confirm that the new value works in your setup by invoking
' lotus - miner proving compute window - post 0 ' ` ,
} ,
{
Name : "MaxPartitionsPerPoStMessage" ,
Type : "int" ,
Comment : ` Maximum number of partitions to prove in a single SubmitWindowPoSt messace . 0 = network limit ( 3 in nv21 )
A single partition may contain up to 2349 32 GiB sectors , or 2300 64 GiB sectors .
//
Note that setting this value lower may result in less efficient gas use - more messages will be sent ,
to prove each deadline , resulting in more total gas use ( but each message will have lower gas limit )
Setting this value above the network limit has no effect ` ,
} ,
{
Name : "MaxPartitionsPerRecoveryMessage" ,
Type : "int" ,
Comment : ` In some cases when submitting DeclareFaultsRecovered messages ,
there may be too many recoveries to fit in a BlockGasLimit .
In those cases it may be necessary to set this value to something low ( eg 1 ) ;
Note that setting this value lower may result in less efficient gas use - more messages will be sent than needed ,
resulting in more total gas use ( but each message will have lower gas limit ) ` ,
} ,
{
Name : "SingleRecoveringPartitionPerPostMessage" ,
Type : "bool" ,
Comment : ` Enable single partition per PoSt Message for partitions containing recovery sectors
In cases when submitting PoSt messages which contain recovering sectors , the default network limit may still be
too high to fit in the block gas limit . In those cases , it becomes useful to only house the single partition
with recovering sectors in the post message
Note that setting this value lower may result in less efficient gas use - more messages will be sent ,
to prove each deadline , resulting in more total gas use ( but each message will have lower gas limit ) ` ,
} ,
} ,
2024-03-15 21:38:13 +00:00
"CurioSubsystemsConfig" : {
{
Name : "EnableWindowPost" ,
Type : "bool" ,
Comment : ` EnableWindowPost enables window post to be executed on this lotus - provider instance . Each machine in the cluster
with WindowPoSt enabled will also participate in the window post scheduler . It is possible to have multiple
machines with WindowPoSt enabled which will provide redundancy , and in case of multiple partitions per deadline ,
will allow for parallel processing of partitions .
It is possible to have instances handling both WindowPoSt and WinningPoSt , which can provide redundancy without
the need for additional machines . In setups like this it is generally recommended to run
partitionsPerDeadline + 1 machines . ` ,
} ,
{
Name : "WindowPostMaxTasks" ,
Type : "int" ,
Comment : ` ` ,
} ,
{
Name : "EnableWinningPost" ,
Type : "bool" ,
Comment : ` EnableWinningPost enables winning post to be executed on this lotus - provider instance .
Each machine in the cluster with WinningPoSt enabled will also participate in the winning post scheduler .
It is possible to mix machines with WindowPoSt and WinningPoSt enabled , for details see the EnableWindowPost
documentation . ` ,
} ,
{
Name : "WinningPostMaxTasks" ,
Type : "int" ,
Comment : ` ` ,
} ,
2024-03-17 16:40:56 +00:00
{
Name : "EnableParkPiece" ,
Type : "bool" ,
Comment : ` EnableParkPiece enables the "piece parking" task to run on this node . This task is responsible for fetching
pieces from the network and storing them in the storage subsystem until sectors are sealed . This task is
only applicable when integrating with boost , and should be enabled on nodes which will hold deal data
from boost until sectors containing the related pieces have the TreeD / TreeR constructed .
Note that future Curio implementations will have a separate task type for fetching pieces from the internet . ` ,
} ,
{
Name : "ParkPieceMaxTasks" ,
Type : "int" ,
Comment : ` ` ,
} ,
2024-03-15 21:38:13 +00:00
{
Name : "EnableSealSDR" ,
Type : "bool" ,
Comment : ` EnableSealSDR enables SDR tasks to run . SDR is the long sequential computation
creating 11 layer files in sector cache directory .
SDR is the first task in the sealing pipeline . It ' s inputs are just the hash of the
unsealed data ( CommD ) , sector number , miner id , and the seal proof type .
It ' s outputs are the 11 layer files in the sector cache directory .
In lotus - miner this was run as part of PreCommit1 . ` ,
} ,
{
Name : "SealSDRMaxTasks" ,
Type : "int" ,
Comment : ` The maximum amount of SDR tasks that can run simultaneously . Note that the maximum number of tasks will
also be bounded by resources available on the machine . ` ,
} ,
{
Name : "EnableSealSDRTrees" ,
Type : "bool" ,
Comment : ` EnableSealSDRTrees enables the SDR pipeline tree - building task to run .
This task handles encoding of unsealed data into last sdr layer and building
of TreeR , TreeC and TreeD .
This task runs after SDR
TreeD is first computed with optional input of unsealed data
TreeR is computed from replica , which is first computed as field
addition of the last SDR layer and the bottom layer of TreeD ( which is the unsealed data )
TreeC is computed from the 11 SDR layers
The 3 trees will later be used to compute the PoRep proof .
In case of SyntheticPoRep challenges for PoRep will be pre - generated at this step , and trees and layers
will be dropped . SyntheticPoRep works by pre - generating a very large set of challenges ( ~ 30 GiB on disk )
then using a small subset of them for the actual PoRep computation . This allows for significant scratch space
saving between PreCommit and PoRep generation at the expense of more computation ( generating challenges in this step )
In lotus - miner this was run as part of PreCommit2 ( TreeD was run in PreCommit1 ) .
Note that nodes with SDRTrees enabled will also answer to Finalize tasks ,
which just remove unneeded tree data after PoRep is computed . ` ,
} ,
{
Name : "SealSDRTreesMaxTasks" ,
Type : "int" ,
Comment : ` The maximum amount of SealSDRTrees tasks that can run simultaneously . Note that the maximum number of tasks will
also be bounded by resources available on the machine . ` ,
} ,
{
Name : "FinalizeMaxTasks" ,
Type : "int" ,
Comment : ` FinalizeMaxTasks is the maximum amount of finalize tasks that can run simultaneously .
The finalize task is enabled on all machines which also handle SDRTrees tasks . Finalize ALWAYS runs on whichever
machine holds sector cache files , as it removes unneeded tree data after PoRep is computed .
Finalize will run in parallel with the SubmitCommitMsg task . ` ,
} ,
{
Name : "EnableSendPrecommitMsg" ,
Type : "bool" ,
Comment : ` EnableSendPrecommitMsg enables the sending of precommit messages to the chain
from this lotus - provider instance .
This runs after SDRTrees and uses the output CommD / CommR ( roots of TreeD / TreeR ) for the message ` ,
} ,
{
Name : "EnablePoRepProof" ,
Type : "bool" ,
Comment : ` EnablePoRepProof enables the computation of the porep proof
This task runs after interactive - porep seed becomes available , which happens 150 epochs ( 75 min ) after the
precommit message lands on chain . This task should run on a machine with a GPU . Vanilla PoRep proofs are
requested from the machine which holds sector cache files which most likely is the machine which ran the SDRTrees
task .
In lotus - miner this was Commit1 / Commit2 ` ,
} ,
{
Name : "PoRepProofMaxTasks" ,
Type : "int" ,
Comment : ` The maximum amount of PoRepProof tasks that can run simultaneously . Note that the maximum number of tasks will
also be bounded by resources available on the machine . ` ,
} ,
{
Name : "EnableSendCommitMsg" ,
Type : "bool" ,
Comment : ` EnableSendCommitMsg enables the sending of commit messages to the chain
from this lotus - provider instance . ` ,
} ,
{
Name : "EnableMoveStorage" ,
Type : "bool" ,
Comment : ` EnableMoveStorage enables the move - into - long - term - storage task to run on this lotus - provider instance .
This tasks should only be enabled on nodes with long - term storage .
The MoveStorage task is the last task in the sealing pipeline . It moves the sealed sector data from the
SDRTrees machine into long - term storage . This task runs after the Finalize task . ` ,
} ,
{
Name : "MoveStorageMaxTasks" ,
Type : "int" ,
Comment : ` The maximum amount of MoveStorage tasks that can run simultaneously . Note that the maximum number of tasks will
also be bounded by resources available on the machine . It is recommended that this value is set to a number which
uses all available network ( or disk ) bandwidth on the machine without causing bottlenecks . ` ,
} ,
{
Name : "EnableWebGui" ,
Type : "bool" ,
Comment : ` EnableWebGui enables the web GUI on this lotus - provider instance . The UI has minimal local overhead , but it should
only need to be run on a single machine in the cluster . ` ,
} ,
{
Name : "GuiAddress" ,
Type : "string" ,
Comment : ` The address that should listen for Web GUI requests. ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"DAGStoreConfig" : {
integrate DAG store and CARv2 in deal-making (#6671)
This commit removes badger from the deal-making processes, and
moves to a new architecture with the dagstore as the cental
component on the miner-side, and CARv2s on the client-side.
Every deal that has been handed off to the sealing subsystem becomes
a shard in the dagstore. Shards are mounted via the LotusMount, which
teaches the dagstore how to load the related piece when serving
retrievals.
When the miner starts the Lotus for the first time with this patch,
we will perform a one-time migration of all active deals into the
dagstore. This is a lightweight process, and it consists simply
of registering the shards in the dagstore.
Shards are backed by the unsealed copy of the piece. This is currently
a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so
when it's time to acquire a shard to serve a retrieval, the unsealed
CARv1 is joined with its index (safeguarded by the dagstore), to form
a read-only blockstore, thus taking the place of the monolithic
badger.
Data transfers have been adjusted to interface directly with CARv2 files.
On inbound transfers (client retrievals, miner storage deals), we stream
the received data into a CARv2 ReadWrite blockstore. On outbound transfers
(client storage deals, miner retrievals), we serve the data off a CARv2
ReadOnly blockstore.
Client-side imports are managed by the refactored *imports.Manager
component (when not using IPFS integration). Just like it before, we use
the go-filestore library to avoid duplicating the data from the original
file in the resulting UnixFS DAG (concretely the leaves). However, the
target of those imports are what we call "ref-CARv2s": CARv2 files placed
under the `$LOTUS_PATH/imports` directory, containing the intermediate
nodes in full, and the leaves as positional references to the original file
on disk.
Client-side retrievals are placed into CARv2 files in the location:
`$LOTUS_PATH/retrievals`.
A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore`
subcommands have been introduced on the miner-side to inspect and manage
the dagstore.
Despite moving to a CARv2-backed system, the IPFS integration has been
respected, and it continues to be possible to make storage deals with data
held in an IPFS node, and to perform retrievals directly into an IPFS node.
NOTE: because the "staging" and "client" Badger blockstores are no longer
used, existing imports on the client will be rendered useless. On startup,
Lotus will enumerate all imports and print WARN statements on the log for
each import that needs to be reimported. These log lines contain these
messages:
- import lacks carv2 path; import will not work; please reimport
- import has missing/broken carv2; please reimport
At the end, we will print a "sanity check completed" message indicating
the count of imports found, and how many were deemed broken.
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "RootDir" ,
integrate DAG store and CARv2 in deal-making (#6671)
This commit removes badger from the deal-making processes, and
moves to a new architecture with the dagstore as the cental
component on the miner-side, and CARv2s on the client-side.
Every deal that has been handed off to the sealing subsystem becomes
a shard in the dagstore. Shards are mounted via the LotusMount, which
teaches the dagstore how to load the related piece when serving
retrievals.
When the miner starts the Lotus for the first time with this patch,
we will perform a one-time migration of all active deals into the
dagstore. This is a lightweight process, and it consists simply
of registering the shards in the dagstore.
Shards are backed by the unsealed copy of the piece. This is currently
a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so
when it's time to acquire a shard to serve a retrieval, the unsealed
CARv1 is joined with its index (safeguarded by the dagstore), to form
a read-only blockstore, thus taking the place of the monolithic
badger.
Data transfers have been adjusted to interface directly with CARv2 files.
On inbound transfers (client retrievals, miner storage deals), we stream
the received data into a CARv2 ReadWrite blockstore. On outbound transfers
(client storage deals, miner retrievals), we serve the data off a CARv2
ReadOnly blockstore.
Client-side imports are managed by the refactored *imports.Manager
component (when not using IPFS integration). Just like it before, we use
the go-filestore library to avoid duplicating the data from the original
file in the resulting UnixFS DAG (concretely the leaves). However, the
target of those imports are what we call "ref-CARv2s": CARv2 files placed
under the `$LOTUS_PATH/imports` directory, containing the intermediate
nodes in full, and the leaves as positional references to the original file
on disk.
Client-side retrievals are placed into CARv2 files in the location:
`$LOTUS_PATH/retrievals`.
A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore`
subcommands have been introduced on the miner-side to inspect and manage
the dagstore.
Despite moving to a CARv2-backed system, the IPFS integration has been
respected, and it continues to be possible to make storage deals with data
held in an IPFS node, and to perform retrievals directly into an IPFS node.
NOTE: because the "staging" and "client" Badger blockstores are no longer
used, existing imports on the client will be rendered useless. On startup,
Lotus will enumerate all imports and print WARN statements on the log for
each import that needs to be reimported. These log lines contain these
messages:
- import lacks carv2 path; import will not work; please reimport
- import has missing/broken carv2; please reimport
At the end, we will print a "sanity check completed" message indicating
the count of imports found, and how many were deemed broken.
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
Type : "string" ,
Comment : ` Path to the dagstore root directory . This directory contains three
subdirectories , which can be symlinked to alternative locations if
need be :
- . / transients : caches unsealed deals that have been fetched from the
storage subsystem for serving retrievals .
- . / indices : stores shard indices .
- . / datastore : holds the KV store tracking the state of every shard
known to the DAG store .
Default value : < LOTUS_MARKETS_PATH > / dagstore ( split deployment ) or
< LOTUS_MINER_PATH > / dagstore ( monolith deployment ) ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxConcurrentIndex" ,
integrate DAG store and CARv2 in deal-making (#6671)
This commit removes badger from the deal-making processes, and
moves to a new architecture with the dagstore as the cental
component on the miner-side, and CARv2s on the client-side.
Every deal that has been handed off to the sealing subsystem becomes
a shard in the dagstore. Shards are mounted via the LotusMount, which
teaches the dagstore how to load the related piece when serving
retrievals.
When the miner starts the Lotus for the first time with this patch,
we will perform a one-time migration of all active deals into the
dagstore. This is a lightweight process, and it consists simply
of registering the shards in the dagstore.
Shards are backed by the unsealed copy of the piece. This is currently
a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so
when it's time to acquire a shard to serve a retrieval, the unsealed
CARv1 is joined with its index (safeguarded by the dagstore), to form
a read-only blockstore, thus taking the place of the monolithic
badger.
Data transfers have been adjusted to interface directly with CARv2 files.
On inbound transfers (client retrievals, miner storage deals), we stream
the received data into a CARv2 ReadWrite blockstore. On outbound transfers
(client storage deals, miner retrievals), we serve the data off a CARv2
ReadOnly blockstore.
Client-side imports are managed by the refactored *imports.Manager
component (when not using IPFS integration). Just like it before, we use
the go-filestore library to avoid duplicating the data from the original
file in the resulting UnixFS DAG (concretely the leaves). However, the
target of those imports are what we call "ref-CARv2s": CARv2 files placed
under the `$LOTUS_PATH/imports` directory, containing the intermediate
nodes in full, and the leaves as positional references to the original file
on disk.
Client-side retrievals are placed into CARv2 files in the location:
`$LOTUS_PATH/retrievals`.
A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore`
subcommands have been introduced on the miner-side to inspect and manage
the dagstore.
Despite moving to a CARv2-backed system, the IPFS integration has been
respected, and it continues to be possible to make storage deals with data
held in an IPFS node, and to perform retrievals directly into an IPFS node.
NOTE: because the "staging" and "client" Badger blockstores are no longer
used, existing imports on the client will be rendered useless. On startup,
Lotus will enumerate all imports and print WARN statements on the log for
each import that needs to be reimported. These log lines contain these
messages:
- import lacks carv2 path; import will not work; please reimport
- import has missing/broken carv2; please reimport
At the end, we will print a "sanity check completed" message indicating
the count of imports found, and how many were deemed broken.
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
Type : "int" ,
Comment : ` The maximum amount of indexing jobs that can run simultaneously .
0 means unlimited .
Default value : 5. ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxConcurrentReadyFetches" ,
integrate DAG store and CARv2 in deal-making (#6671)
This commit removes badger from the deal-making processes, and
moves to a new architecture with the dagstore as the cental
component on the miner-side, and CARv2s on the client-side.
Every deal that has been handed off to the sealing subsystem becomes
a shard in the dagstore. Shards are mounted via the LotusMount, which
teaches the dagstore how to load the related piece when serving
retrievals.
When the miner starts the Lotus for the first time with this patch,
we will perform a one-time migration of all active deals into the
dagstore. This is a lightweight process, and it consists simply
of registering the shards in the dagstore.
Shards are backed by the unsealed copy of the piece. This is currently
a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so
when it's time to acquire a shard to serve a retrieval, the unsealed
CARv1 is joined with its index (safeguarded by the dagstore), to form
a read-only blockstore, thus taking the place of the monolithic
badger.
Data transfers have been adjusted to interface directly with CARv2 files.
On inbound transfers (client retrievals, miner storage deals), we stream
the received data into a CARv2 ReadWrite blockstore. On outbound transfers
(client storage deals, miner retrievals), we serve the data off a CARv2
ReadOnly blockstore.
Client-side imports are managed by the refactored *imports.Manager
component (when not using IPFS integration). Just like it before, we use
the go-filestore library to avoid duplicating the data from the original
file in the resulting UnixFS DAG (concretely the leaves). However, the
target of those imports are what we call "ref-CARv2s": CARv2 files placed
under the `$LOTUS_PATH/imports` directory, containing the intermediate
nodes in full, and the leaves as positional references to the original file
on disk.
Client-side retrievals are placed into CARv2 files in the location:
`$LOTUS_PATH/retrievals`.
A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore`
subcommands have been introduced on the miner-side to inspect and manage
the dagstore.
Despite moving to a CARv2-backed system, the IPFS integration has been
respected, and it continues to be possible to make storage deals with data
held in an IPFS node, and to perform retrievals directly into an IPFS node.
NOTE: because the "staging" and "client" Badger blockstores are no longer
used, existing imports on the client will be rendered useless. On startup,
Lotus will enumerate all imports and print WARN statements on the log for
each import that needs to be reimported. These log lines contain these
messages:
- import lacks carv2 path; import will not work; please reimport
- import has missing/broken carv2; please reimport
At the end, we will print a "sanity check completed" message indicating
the count of imports found, and how many were deemed broken.
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
Type : "int" ,
Comment : ` The maximum amount of unsealed deals that can be fetched simultaneously
from the storage subsystem . 0 means unlimited .
2022-01-13 18:26:13 +00:00
Default value : 0 ( unlimited ) . ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxConcurrentUnseals" ,
2022-01-13 18:26:13 +00:00
Type : "int" ,
Comment : ` The maximum amount of unseals that can be processed simultaneously
from the storage subsystem . 0 means unlimited .
integrate DAG store and CARv2 in deal-making (#6671)
This commit removes badger from the deal-making processes, and
moves to a new architecture with the dagstore as the cental
component on the miner-side, and CARv2s on the client-side.
Every deal that has been handed off to the sealing subsystem becomes
a shard in the dagstore. Shards are mounted via the LotusMount, which
teaches the dagstore how to load the related piece when serving
retrievals.
When the miner starts the Lotus for the first time with this patch,
we will perform a one-time migration of all active deals into the
dagstore. This is a lightweight process, and it consists simply
of registering the shards in the dagstore.
Shards are backed by the unsealed copy of the piece. This is currently
a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so
when it's time to acquire a shard to serve a retrieval, the unsealed
CARv1 is joined with its index (safeguarded by the dagstore), to form
a read-only blockstore, thus taking the place of the monolithic
badger.
Data transfers have been adjusted to interface directly with CARv2 files.
On inbound transfers (client retrievals, miner storage deals), we stream
the received data into a CARv2 ReadWrite blockstore. On outbound transfers
(client storage deals, miner retrievals), we serve the data off a CARv2
ReadOnly blockstore.
Client-side imports are managed by the refactored *imports.Manager
component (when not using IPFS integration). Just like it before, we use
the go-filestore library to avoid duplicating the data from the original
file in the resulting UnixFS DAG (concretely the leaves). However, the
target of those imports are what we call "ref-CARv2s": CARv2 files placed
under the `$LOTUS_PATH/imports` directory, containing the intermediate
nodes in full, and the leaves as positional references to the original file
on disk.
Client-side retrievals are placed into CARv2 files in the location:
`$LOTUS_PATH/retrievals`.
A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore`
subcommands have been introduced on the miner-side to inspect and manage
the dagstore.
Despite moving to a CARv2-backed system, the IPFS integration has been
respected, and it continues to be possible to make storage deals with data
held in an IPFS node, and to perform retrievals directly into an IPFS node.
NOTE: because the "staging" and "client" Badger blockstores are no longer
used, existing imports on the client will be rendered useless. On startup,
Lotus will enumerate all imports and print WARN statements on the log for
each import that needs to be reimported. These log lines contain these
messages:
- import lacks carv2 path; import will not work; please reimport
- import has missing/broken carv2; please reimport
At the end, we will print a "sanity check completed" message indicating
the count of imports found, and how many were deemed broken.
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
Default value : 0 ( unlimited ) . ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxConcurrencyStorageCalls" ,
integrate DAG store and CARv2 in deal-making (#6671)
This commit removes badger from the deal-making processes, and
moves to a new architecture with the dagstore as the cental
component on the miner-side, and CARv2s on the client-side.
Every deal that has been handed off to the sealing subsystem becomes
a shard in the dagstore. Shards are mounted via the LotusMount, which
teaches the dagstore how to load the related piece when serving
retrievals.
When the miner starts the Lotus for the first time with this patch,
we will perform a one-time migration of all active deals into the
dagstore. This is a lightweight process, and it consists simply
of registering the shards in the dagstore.
Shards are backed by the unsealed copy of the piece. This is currently
a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so
when it's time to acquire a shard to serve a retrieval, the unsealed
CARv1 is joined with its index (safeguarded by the dagstore), to form
a read-only blockstore, thus taking the place of the monolithic
badger.
Data transfers have been adjusted to interface directly with CARv2 files.
On inbound transfers (client retrievals, miner storage deals), we stream
the received data into a CARv2 ReadWrite blockstore. On outbound transfers
(client storage deals, miner retrievals), we serve the data off a CARv2
ReadOnly blockstore.
Client-side imports are managed by the refactored *imports.Manager
component (when not using IPFS integration). Just like it before, we use
the go-filestore library to avoid duplicating the data from the original
file in the resulting UnixFS DAG (concretely the leaves). However, the
target of those imports are what we call "ref-CARv2s": CARv2 files placed
under the `$LOTUS_PATH/imports` directory, containing the intermediate
nodes in full, and the leaves as positional references to the original file
on disk.
Client-side retrievals are placed into CARv2 files in the location:
`$LOTUS_PATH/retrievals`.
A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore`
subcommands have been introduced on the miner-side to inspect and manage
the dagstore.
Despite moving to a CARv2-backed system, the IPFS integration has been
respected, and it continues to be possible to make storage deals with data
held in an IPFS node, and to perform retrievals directly into an IPFS node.
NOTE: because the "staging" and "client" Badger blockstores are no longer
used, existing imports on the client will be rendered useless. On startup,
Lotus will enumerate all imports and print WARN statements on the log for
each import that needs to be reimported. These log lines contain these
messages:
- import lacks carv2 path; import will not work; please reimport
- import has missing/broken carv2; please reimport
At the end, we will print a "sanity check completed" message indicating
the count of imports found, and how many were deemed broken.
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
Type : "int" ,
Comment : ` The maximum number of simultaneous inflight API calls to the storage
subsystem .
Default value : 100. ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "GCInterval" ,
integrate DAG store and CARv2 in deal-making (#6671)
This commit removes badger from the deal-making processes, and
moves to a new architecture with the dagstore as the cental
component on the miner-side, and CARv2s on the client-side.
Every deal that has been handed off to the sealing subsystem becomes
a shard in the dagstore. Shards are mounted via the LotusMount, which
teaches the dagstore how to load the related piece when serving
retrievals.
When the miner starts the Lotus for the first time with this patch,
we will perform a one-time migration of all active deals into the
dagstore. This is a lightweight process, and it consists simply
of registering the shards in the dagstore.
Shards are backed by the unsealed copy of the piece. This is currently
a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so
when it's time to acquire a shard to serve a retrieval, the unsealed
CARv1 is joined with its index (safeguarded by the dagstore), to form
a read-only blockstore, thus taking the place of the monolithic
badger.
Data transfers have been adjusted to interface directly with CARv2 files.
On inbound transfers (client retrievals, miner storage deals), we stream
the received data into a CARv2 ReadWrite blockstore. On outbound transfers
(client storage deals, miner retrievals), we serve the data off a CARv2
ReadOnly blockstore.
Client-side imports are managed by the refactored *imports.Manager
component (when not using IPFS integration). Just like it before, we use
the go-filestore library to avoid duplicating the data from the original
file in the resulting UnixFS DAG (concretely the leaves). However, the
target of those imports are what we call "ref-CARv2s": CARv2 files placed
under the `$LOTUS_PATH/imports` directory, containing the intermediate
nodes in full, and the leaves as positional references to the original file
on disk.
Client-side retrievals are placed into CARv2 files in the location:
`$LOTUS_PATH/retrievals`.
A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore`
subcommands have been introduced on the miner-side to inspect and manage
the dagstore.
Despite moving to a CARv2-backed system, the IPFS integration has been
respected, and it continues to be possible to make storage deals with data
held in an IPFS node, and to perform retrievals directly into an IPFS node.
NOTE: because the "staging" and "client" Badger blockstores are no longer
used, existing imports on the client will be rendered useless. On startup,
Lotus will enumerate all imports and print WARN statements on the log for
each import that needs to be reimported. These log lines contain these
messages:
- import lacks carv2 path; import will not work; please reimport
- import has missing/broken carv2; please reimport
At the end, we will print a "sanity check completed" message indicating
the count of imports found, and how many were deemed broken.
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
Type : "Duration" ,
Comment : ` The time between calls to periodic dagstore GC , in time . Duration string
representation , e . g . 1 m , 5 m , 1 h .
Default value : 1 minute . ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"DealmakingConfig" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "ConsiderOnlineStorageDeals" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 13:40:30 +00:00
Comment : ` When enabled, the miner can accept online deals ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "ConsiderOfflineStorageDeals" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 13:40:30 +00:00
Comment : ` When enabled, the miner can accept offline deals ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "ConsiderOnlineRetrievalDeals" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 13:40:30 +00:00
Comment : ` When enabled, the miner can accept retrieval deals ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "ConsiderOfflineRetrievalDeals" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 13:40:30 +00:00
Comment : ` When enabled, the miner can accept offline retrieval deals ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "ConsiderVerifiedStorageDeals" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 13:40:30 +00:00
Comment : ` When enabled, the miner can accept verified deals ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "ConsiderUnverifiedStorageDeals" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 13:40:30 +00:00
Comment : ` When enabled, the miner can accept unverified deals ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "PieceCidBlocklist" ,
2021-07-23 13:18:20 +00:00
Type : "[]cid.Cid" ,
2021-07-23 13:40:30 +00:00
Comment : ` A list of Data CIDs to reject when making deals ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "ExpectedSealDuration" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 13:40:30 +00:00
Comment : ` Maximum expected amount of time getting the deal into a sealed sector will take
This includes the time the deal will need to get transferred and published
before being assigned to a sector ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxDealStartDelay" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 11:55:50 +00:00
Comment : ` Maximum amount of time proposed deal StartEpoch can be in future ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "PublishMsgPeriod" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 13:40:30 +00:00
Comment : ` When a deal is ready to publish , the amount of time to wait for more
deals to be ready to publish before publishing them all as a batch ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxDealsPerPublishMsg" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2021-07-23 11:55:50 +00:00
Comment : ` The maximum number of deals to include in a single PublishStorageDeals
message ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxProviderCollateralMultiplier" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2021-07-23 11:55:50 +00:00
Comment : ` The maximum collateral that the provider will put up against a deal ,
as a multiplier of the minimum collateral bound ` ,
2021-09-06 13:00:17 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxStagingDealsBytes" ,
2021-09-06 13:00:17 +00:00
Type : "int64" ,
2021-09-06 13:52:25 +00:00
Comment : ` The maximum allowed disk usage size in bytes of staging deals not yet
2021-09-06 15:07:19 +00:00
passed to the sealing node by the markets service . 0 is unlimited . ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "SimultaneousTransfersForStorage" ,
2021-09-30 01:49:59 +00:00
Type : "uint64" ,
Comment : ` The maximum number of parallel online data transfers for storage deals ` ,
} ,
2021-10-28 11:39:57 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "SimultaneousTransfersForStoragePerClient" ,
2021-10-28 11:39:57 +00:00
Type : "uint64" ,
Comment : ` The maximum number of simultaneous data transfers from any single client
for storage deals .
Unset by default ( 0 ) , and values higher than SimultaneousTransfersForStorage
will have no effect ; i . e . the total number of simultaneous data transfers
across all storage clients is bound by SimultaneousTransfersForStorage
regardless of this number . ` ,
} ,
2021-09-30 01:49:59 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "SimultaneousTransfersForRetrieval" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2021-09-30 01:49:59 +00:00
Comment : ` The maximum number of parallel online data transfers for retrieval deals ` ,
2021-07-23 11:55:50 +00:00
} ,
2021-09-30 12:35:23 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "StartEpochSealingBuffer" ,
2021-09-30 12:35:23 +00:00
Type : "uint64" ,
Comment : ` Minimum start epoch buffer to give time for sealing of sector with deal. ` ,
} ,
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Filter" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 13:40:30 +00:00
Comment : ` A command used for fine - grained evaluation of storage deals
2022-09-13 23:10:22 +00:00
see https : //lotus.filecoin.io/storage-providers/advanced-configurations/market/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "RetrievalFilter" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 13:40:30 +00:00
Comment : ` A command used for fine - grained evaluation of retrieval deals
2022-09-13 23:10:22 +00:00
see https : //lotus.filecoin.io/storage-providers/advanced-configurations/market/#using-filters-for-fine-grained-storage-and-retrieval-deal-acceptance for more details`,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "RetrievalPricing" ,
2021-07-23 13:18:20 +00:00
Type : "*RetrievalPricing" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
2024-03-08 07:43:39 +00:00
"EventsConfig" : {
2023-01-19 22:44:58 +00:00
{
Name : "DisableRealTimeFilterAPI" ,
Type : "bool" ,
chore: Merge nv22 into master (#11699)
* [WIP] feat: Add nv22 skeleton
Addition of Network Version 22 skeleton
* update FFI
* feat: drand: refactor round verification
* feat: sealing: Support nv22 DDO features in the sealing pipeline (#11226)
* Initial work supporting DDO pieces in lotus-miner
* sealing: Update pipeline input to operate on UniversalPiece
* sealing: Update pipeline checks/sealing states to operate on UniversalPiece
* sealing: Make pipeline build with UniversalPiece
* move PieceDealInfo out of api
* make gen
* make sealing pipeline unit tests pass
* fix itest ensemble build
* don't panic in SectorsStatus with deals
* stop linter from complaining about checkPieces
* fix sector import tests
* mod tidy
* sealing: Add logic for (pre)committing DDO sectors
* sealing: state-types with method defs
* DDO non-snap pipeline works(?), DDO Itests
* DDO support in snapdeals pipeline
* make gen
* update actor bundles
* update the gst market fix
* fix: chain: use PreCommitSectorsBatch2 when setting up genesis
* some bug fixes
* integration working changes
* update actor bundles
* Make TestOnboardRawPieceSnap pass
* Appease the linter
* Make deadlines test pass with v12 actors
* Update go-state-types, abstract market DealState
* make gen
* mod tidy, lint fixes
* Fix some more tests
* Bump version in master
Bump version in master
* Make gen
Make gen
* fix sender
* fix: lotus-provider: Fix winning PoSt
* fix: sql Scan cannot write to an object
* Actually show miner-addrs in info-log
Actually show miner-addrs in lotus-provider info-log
* [WIP] feat: Add nv22 skeleton
Addition of Network Version 22 skeleton
* update FFI
* ddo is now nv22
* make gen
* temp actor bundle with ddo
* use working go-state-types
* gst with v13 market migration
* update bundle, builtin.MethodsMiner.ProveCommitSectors2 -> 3
* actually working v13 migration, v13 migration itest
* Address review
* sealing: Correct DDO snap pledge math
* itests: Mixed ddo itest
* pipeline: Fix sectorWeight
* sealing: convert market deals into PAMs in mixed sectors
* sealing: make market to ddo conversion work
* fix lint
* update gst
* Update actors and GST to lastest integ branch
* commit batcher: Update ProveCommitSectors3Params builder logic
* make gen
* use builtin-actors master
* ddo: address review
* itests: Add commd assertions to ddo tests
* make gen
* gst with fixed types
* config knobs for RequireActivationSuccess
* storage: Drop obsolete flaky tasts
---------
Co-authored-by: Jennifer Wang <jiayingw703@gmail.com>
Co-authored-by: Aayush <arajasek94@gmail.com>
Co-authored-by: Shrenuj Bansal <shrenuj.bansal@protocol.ai>
Co-authored-by: Phi <orjan.roren@gmail.com>
Co-authored-by: Andrew Jackson (Ajax) <snadrus@gmail.com>
Co-authored-by: TippyFlits <james.bluett@protocol.ai>
* feat: implement FIP-0063
* chore: deps: update to go-multiaddr v0.12.2 (#11602)
* feat: fvm: update the FVM/FFI to v4.1 (#11608) (#11612)
This:
1. Adds nv22 support.
2. Updates the message tracing format.
Co-authored-by: Steven Allen <steven@stebalien.com>
* AggregateProofType nil when doing batch updates
Use latest nv22 go-state-types version with matching update
* Update to v13.0.0-rc.2 bundle
* chore: Upgrade heights and codename
Update upgrade heights
Co-Authored-By: Steven Allen <steven@stebalien.com>
* Update epoch after nv22 DRAND switch
Update epoch after nv22 DRAND switch
* Update Mango codename to Phoneix
Make the codename for the Drand-change inline with Dragon style.
* Add UpgradePhoenixHeight to API params
* set UpgradePhoenixHeight to be one hour after Dragon
* Make gen
Make gen and UpgradePhoenixHeight in butterfly and local devnet to be in line with Calibration and Mainnet
* Update epoch heights (#11637)
Update epoch heights
* new: add forest bootstrap nodes (#11636)
Signed-off-by: samuelarogbonlo <sbayo971@gmail.com>
* Merge pull request #11491 from filecoin-project/fix/remove-decommissioned-pl-bootstrap-nodes
Remove PL operated bootstrap nodes from mainnet.pi
* feat: api: new verified registry methods to get all allocations and claims (#11631)
* new verireg methods
* update changelog and add itest
* update itest and cli
* update new method's support till v9
* remove gateway APIs
* fix cli internal var names
* chore:: backport #11609 to the feat/nv22 branch (#11644)
* feat: api: improve the correctness of Eth's trace_block (#11609)
* Improve the correctness of Eth's trace_block
- Improve encoding/decoding of parameters and return values:
- Encode "native" parameters and return values with Solidity ABI.
- Correctly decode parameters to "create" calls.
- Use the correct (ish) output for "create" calls.
- Handle all forms of "create".
- Make robust with respect to reverts:
- Use the actor ID/address from the trace instead of looking it up in
the state-tree (may not exist in the state-tree due to a revert).
- Gracefully handle failed actor/contract creation.
- Improve performance:
- We avoid looking anything up in the state-tree when translating the
trace, which should significantly improve performance.
- Improve code readability:
- Remove all "backtracking" logic.
- Use an "environment" struct to store temporary state instead of
attaching it to the trace.
- Fix random bugs:
- Fix an allocation bug in the "address" logic (need to set the
capacity before modifying the slice).
- Improved error checking/handling.
- Use correct types for `trace_block` action/results (create, call, etc.).
- And use the correct types for Result/Action structs instead of reusing the same "Call" action every time.
- Improve error messages.
* Make gen
Make gen
---------
Co-authored-by: Steven Allen <steven@stebalien.com>
* fix: add UpgradePhoenixHeight to StateGetNetworkParams (#11648)
* chore: deps: update to go-state-types v13.0.0-rc.1
* do NOT update the cache when running the real migration
* Merge pull request #11632 from hanabi1224/hm/drand-test
feat: drand quicknet: allow scheduling drand quicknet upgrade before nv22 on 2k devnet
* chore: deps: update to go-state-types v13.0.0-rc.2
chore: deps: update to go-state-types v13.0.0-rc.2
* feat: set migration config UpgradeEpoch for v13 actors upgrade
* Built-in actor events first draft
* itest for DDO non-market verified data w/ builtin actor events
* Tests for builtin actor events API
* Clean up DDO+Events tests, add lots of explainer comments
* Minor tweaks to events types
* Avoid duplicate messages when looking for receipts
* Rename internal events modules for clarity
* Adjust actor event API after review
* s/ActorEvents/Events/g in global config
* Manage event sending rate for SubscribeActorEvents
* Terminate SubscribeActorEvents chan when at max height
* Document future API changes
* More clarity in actor event API docs
* More post-review changes, lots of tests for SubscribeActorEvents
Use BlockDelay as the window for receiving events on the SubscribeActorEvents
channel. We expect the user to have received the initial batch of historical
events (if any) in one block's time. For real-time events we expect them to
not fall behind by roughly one block's time.
* Remove duplicate code from actor event type marshalling tests
Reduce verbosity and remove duplicate test logic from actor event types
JSON marshalling tests.
* Rename actor events test to follow go convention
Add missing `s` to `actor_events` test file to follow golang convention
used across the repo.
* Run actor events table tests in deterministic order
Refactor `map` usage for actor event table tests to ensure deterministic
test execution order, making debugging potential issues easier. If
non-determinism is a target, leverage Go's built-in parallel testing
capabilities.
* Reduce scope for filter removal failure when getting actor events
Use a fresh context to remove the temporary filter installed solely to
get the actor events. This should reduce chances of failure in a case
where the original context may be expired/cancelled.
Refactor removal into a `defer` statement for a more readable, concise
return statement.
* Use fixed RNG seed for actor event tests
Improve determinism in actor event tests by using a fixed RNG seed. This
makes up a more reproducible test suit.
* Use provided libraries to assert eventual conditions
Use the functionalities already provided by `testify` to assert eventual
conditions, and remove the use of `time.Sleep`.
Remove duplicate code in utility functions that are already defined.
Refactor assertion helper functions to use consistent terminology:
"require" implies fatal error, whereas "assert" implies error where the
test may proceed executing.
* Update changelog for actor events APIs
* Fix concerns and docs identified by review
* Update actor bundle to v13.0.0-rc3
Update actor bundle to v13.0.0-rc3
* Prep Lotus v1.26.0-rc1
- For sanity reverting the mainnet upgrade epoch to 99999999, and then only set it when cutting the final release
-Update Calibnet CIDs to v13.0.0-rc3
- Add GetActorEvents, SubscribeActorEvents, GetAllClaims and GetAllAllocations methods to the changelog
Co-Authored-By: Jiaying Wang <42981373+jennijuju@users.noreply.github.com>
* Update CHANGELOG.md
Co-authored-by: Masih H. Derkani <m@derkani.org>
* Make gen
Make gen
* fix: beacon: validate drand change at nv16 correctly
* bump to v1.26.0-rc2
* test: cleanup ddo verified itest, extract steps to functions
also add allocation-removed event case
* test: extract verified DDO test to separate file, add more checks
* test: add additional actor events checks
* Add verification for "deal-activated" actor event
* docs(drand): document the meaning of "IsChained" (#11692)
* Resolve conflicts
I encountered multiple issues when trying to run make gen. And these changes fixed a couple of them:
- go mod tidy
- Remove RaftState/RaftLeader
- Revert `if ts.Height() > claim.TermMax+claim.TermStart || !cctx.IsSet("expired")` to the what is in the release/v1.26.0: `if tsHeight > val.TermMax || !expired`
* fixup imports, make jen
* Update version
Update version in master to v1.27.0-dev
* Update node/impl/full/dummy.go
Co-authored-by: Łukasz Magiera <magik6k@users.noreply.github.com>
* Adjust ListClaimsCmd
Adjust ListClaimsCmd according to review
---------
Signed-off-by: samuelarogbonlo <sbayo971@gmail.com>
Co-authored-by: TippyFlits <james.bluett@protocol.ai>
Co-authored-by: Aayush <arajasek94@gmail.com>
Co-authored-by: Łukasz Magiera <magik6k@users.noreply.github.com>
Co-authored-by: Jennifer Wang <jiayingw703@gmail.com>
Co-authored-by: Shrenuj Bansal <shrenuj.bansal@protocol.ai>
Co-authored-by: Andrew Jackson (Ajax) <snadrus@gmail.com>
Co-authored-by: Steven Allen <steven@stebalien.com>
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Samuel Arogbonlo <47984109+samuelarogbonlo@users.noreply.github.com>
Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com>
Co-authored-by: tom123222 <160735201+tom123222@users.noreply.github.com>
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Masih H. Derkani <m@derkani.org>
Co-authored-by: Jiaying Wang <42981373+jennijuju@users.noreply.github.com>
2024-03-12 09:33:58 +00:00
Comment : ` DisableRealTimeFilterAPI will disable the RealTimeFilterAPI that can create and query filters for actor events as they are emitted .
2024-03-08 07:43:39 +00:00
The API is enabled when Fevm . EnableEthRPC or EnableActorEventsAPI is true , but can be disabled selectively with this flag . ` ,
2023-01-19 22:44:58 +00:00
} ,
{
Name : "DisableHistoricFilterAPI" ,
Type : "bool" ,
Comment : ` DisableHistoricFilterAPI will disable the HistoricFilterAPI that can create and query filters for actor events
that occurred in the past . HistoricFilterAPI maintains a queryable index of events .
2024-03-08 07:43:39 +00:00
The API is enabled when Fevm . EnableEthRPC or EnableActorEventsAPI is true , but can be disabled selectively with this flag . ` ,
} ,
{
Name : "EnableActorEventsAPI" ,
Type : "bool" ,
Comment : ` EnableActorEventsAPI enables the Actor events API that enables clients to consume events
emitted by ( smart contracts + built - in Actors ) .
This will also enable the RealTimeFilterAPI and HistoricFilterAPI by default , but they can be
disabled by setting their respective Disable * options . ` ,
2023-01-19 22:44:58 +00:00
} ,
{
Name : "FilterTTL" ,
Type : "Duration" ,
Comment : ` FilterTTL specifies the time to live for actor event filters . Filters that haven ' t been accessed longer than
this time become eligible for automatic deletion . ` ,
} ,
{
Name : "MaxFilters" ,
Type : "int" ,
Comment : ` MaxFilters specifies the maximum number of filters that may exist at any one time. ` ,
} ,
{
Name : "MaxFilterResults" ,
Type : "int" ,
Comment : ` MaxFilterResults specifies the maximum number of results that can be accumulated by an actor event filter. ` ,
} ,
{
Name : "MaxFilterHeightRange" ,
Type : "uint64" ,
Comment : ` MaxFilterHeightRange specifies the maximum range of heights that can be used in a filter ( to avoid querying
the entire chain ) ` ,
} ,
{
Name : "DatabasePath" ,
Type : "string" ,
Comment : ` DatabasePath is the full path to a sqlite database that will be used to index actor events to
support the historic filter APIs . If the database does not exist it will be created . The directory containing
the database must already exist and be writeable . If a relative path is provided here , sqlite treats it as
relative to the CWD ( current working directory ) . ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"FaultReporterConfig" : {
2023-08-01 15:28:47 +00:00
{
Name : "EnableConsensusFaultReporter" ,
Type : "bool" ,
Comment : ` EnableConsensusFaultReporter controls whether the node will monitor and
report consensus faults . When enabled , the node will watch for malicious
behaviors like double - mining and parent grinding , and submit reports to the
network . This can earn reporter rewards , but is not guaranteed . Nodes should
enable fault reporting with care , as it may increase resource usage , and may
generate gas fees without earning rewards . ` ,
} ,
{
Name : "ConsensusFaultReporterDataDir" ,
Type : "string" ,
Comment : ` ConsensusFaultReporterDataDir is the path where fault reporter state will be
persisted . This directory should have adequate space and permissions for the
node process . ` ,
} ,
{
Name : "ConsensusFaultReporterAddress" ,
Type : "string" ,
Comment : ` ConsensusFaultReporterAddress is the wallet address used for submitting
ReportConsensusFault messages . It will pay for gas fees , and receive any
rewards . This address should have adequate funds to cover gas fees . ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"FeeConfig" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "DefaultMaxFee" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"FevmConfig" : {
2023-01-13 16:49:01 +00:00
{
2023-01-19 20:35:19 +00:00
Name : "EnableEthRPC" ,
2023-01-13 16:49:01 +00:00
Type : "bool" ,
2023-01-19 22:44:58 +00:00
Comment : ` EnableEthRPC enables eth_ rpc , and enables storing a mapping of eth transaction hashes to filecoin message Cids .
This will also enable the RealTimeFilterAPI and HistoricFilterAPI by default , but they can be disabled by config options above . ` ,
2023-01-13 16:49:01 +00:00
} ,
2023-01-16 07:56:45 +00:00
{
Name : "EthTxHashMappingLifetimeDays" ,
Type : "int" ,
Comment : ` EthTxHashMappingLifetimeDays the transaction hash lookup database will delete mappings that have been stored for more than x days
Set to 0 to keep all mappings ` ,
} ,
2023-01-19 22:44:58 +00:00
{
Name : "Events" ,
2024-03-08 07:43:39 +00:00
Type : "DeprecatedEvents" ,
2023-01-19 22:44:58 +00:00
Comment : ` ` ,
} ,
2023-01-13 16:49:01 +00:00
} ,
2023-09-22 21:15:18 +00:00
"FullNode" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Client" ,
2021-07-23 13:18:20 +00:00
Type : "Client" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Wallet" ,
2021-07-23 13:18:20 +00:00
Type : "Wallet" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Fees" ,
2021-07-23 13:18:20 +00:00
Type : "FeeConfig" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Chainstore" ,
2021-07-23 13:18:20 +00:00
Type : "Chainstore" ,
2022-09-13 17:05:48 +00:00
Comment : ` ` ,
} ,
2023-01-04 13:22:41 +00:00
{
2023-01-13 16:49:01 +00:00
Name : "Fevm" ,
Type : "FevmConfig" ,
2023-01-04 13:22:41 +00:00
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
chore: Merge nv22 into master (#11699)
* [WIP] feat: Add nv22 skeleton
Addition of Network Version 22 skeleton
* update FFI
* feat: drand: refactor round verification
* feat: sealing: Support nv22 DDO features in the sealing pipeline (#11226)
* Initial work supporting DDO pieces in lotus-miner
* sealing: Update pipeline input to operate on UniversalPiece
* sealing: Update pipeline checks/sealing states to operate on UniversalPiece
* sealing: Make pipeline build with UniversalPiece
* move PieceDealInfo out of api
* make gen
* make sealing pipeline unit tests pass
* fix itest ensemble build
* don't panic in SectorsStatus with deals
* stop linter from complaining about checkPieces
* fix sector import tests
* mod tidy
* sealing: Add logic for (pre)committing DDO sectors
* sealing: state-types with method defs
* DDO non-snap pipeline works(?), DDO Itests
* DDO support in snapdeals pipeline
* make gen
* update actor bundles
* update the gst market fix
* fix: chain: use PreCommitSectorsBatch2 when setting up genesis
* some bug fixes
* integration working changes
* update actor bundles
* Make TestOnboardRawPieceSnap pass
* Appease the linter
* Make deadlines test pass with v12 actors
* Update go-state-types, abstract market DealState
* make gen
* mod tidy, lint fixes
* Fix some more tests
* Bump version in master
Bump version in master
* Make gen
Make gen
* fix sender
* fix: lotus-provider: Fix winning PoSt
* fix: sql Scan cannot write to an object
* Actually show miner-addrs in info-log
Actually show miner-addrs in lotus-provider info-log
* [WIP] feat: Add nv22 skeleton
Addition of Network Version 22 skeleton
* update FFI
* ddo is now nv22
* make gen
* temp actor bundle with ddo
* use working go-state-types
* gst with v13 market migration
* update bundle, builtin.MethodsMiner.ProveCommitSectors2 -> 3
* actually working v13 migration, v13 migration itest
* Address review
* sealing: Correct DDO snap pledge math
* itests: Mixed ddo itest
* pipeline: Fix sectorWeight
* sealing: convert market deals into PAMs in mixed sectors
* sealing: make market to ddo conversion work
* fix lint
* update gst
* Update actors and GST to lastest integ branch
* commit batcher: Update ProveCommitSectors3Params builder logic
* make gen
* use builtin-actors master
* ddo: address review
* itests: Add commd assertions to ddo tests
* make gen
* gst with fixed types
* config knobs for RequireActivationSuccess
* storage: Drop obsolete flaky tasts
---------
Co-authored-by: Jennifer Wang <jiayingw703@gmail.com>
Co-authored-by: Aayush <arajasek94@gmail.com>
Co-authored-by: Shrenuj Bansal <shrenuj.bansal@protocol.ai>
Co-authored-by: Phi <orjan.roren@gmail.com>
Co-authored-by: Andrew Jackson (Ajax) <snadrus@gmail.com>
Co-authored-by: TippyFlits <james.bluett@protocol.ai>
* feat: implement FIP-0063
* chore: deps: update to go-multiaddr v0.12.2 (#11602)
* feat: fvm: update the FVM/FFI to v4.1 (#11608) (#11612)
This:
1. Adds nv22 support.
2. Updates the message tracing format.
Co-authored-by: Steven Allen <steven@stebalien.com>
* AggregateProofType nil when doing batch updates
Use latest nv22 go-state-types version with matching update
* Update to v13.0.0-rc.2 bundle
* chore: Upgrade heights and codename
Update upgrade heights
Co-Authored-By: Steven Allen <steven@stebalien.com>
* Update epoch after nv22 DRAND switch
Update epoch after nv22 DRAND switch
* Update Mango codename to Phoneix
Make the codename for the Drand-change inline with Dragon style.
* Add UpgradePhoenixHeight to API params
* set UpgradePhoenixHeight to be one hour after Dragon
* Make gen
Make gen and UpgradePhoenixHeight in butterfly and local devnet to be in line with Calibration and Mainnet
* Update epoch heights (#11637)
Update epoch heights
* new: add forest bootstrap nodes (#11636)
Signed-off-by: samuelarogbonlo <sbayo971@gmail.com>
* Merge pull request #11491 from filecoin-project/fix/remove-decommissioned-pl-bootstrap-nodes
Remove PL operated bootstrap nodes from mainnet.pi
* feat: api: new verified registry methods to get all allocations and claims (#11631)
* new verireg methods
* update changelog and add itest
* update itest and cli
* update new method's support till v9
* remove gateway APIs
* fix cli internal var names
* chore:: backport #11609 to the feat/nv22 branch (#11644)
* feat: api: improve the correctness of Eth's trace_block (#11609)
* Improve the correctness of Eth's trace_block
- Improve encoding/decoding of parameters and return values:
- Encode "native" parameters and return values with Solidity ABI.
- Correctly decode parameters to "create" calls.
- Use the correct (ish) output for "create" calls.
- Handle all forms of "create".
- Make robust with respect to reverts:
- Use the actor ID/address from the trace instead of looking it up in
the state-tree (may not exist in the state-tree due to a revert).
- Gracefully handle failed actor/contract creation.
- Improve performance:
- We avoid looking anything up in the state-tree when translating the
trace, which should significantly improve performance.
- Improve code readability:
- Remove all "backtracking" logic.
- Use an "environment" struct to store temporary state instead of
attaching it to the trace.
- Fix random bugs:
- Fix an allocation bug in the "address" logic (need to set the
capacity before modifying the slice).
- Improved error checking/handling.
- Use correct types for `trace_block` action/results (create, call, etc.).
- And use the correct types for Result/Action structs instead of reusing the same "Call" action every time.
- Improve error messages.
* Make gen
Make gen
---------
Co-authored-by: Steven Allen <steven@stebalien.com>
* fix: add UpgradePhoenixHeight to StateGetNetworkParams (#11648)
* chore: deps: update to go-state-types v13.0.0-rc.1
* do NOT update the cache when running the real migration
* Merge pull request #11632 from hanabi1224/hm/drand-test
feat: drand quicknet: allow scheduling drand quicknet upgrade before nv22 on 2k devnet
* chore: deps: update to go-state-types v13.0.0-rc.2
chore: deps: update to go-state-types v13.0.0-rc.2
* feat: set migration config UpgradeEpoch for v13 actors upgrade
* Built-in actor events first draft
* itest for DDO non-market verified data w/ builtin actor events
* Tests for builtin actor events API
* Clean up DDO+Events tests, add lots of explainer comments
* Minor tweaks to events types
* Avoid duplicate messages when looking for receipts
* Rename internal events modules for clarity
* Adjust actor event API after review
* s/ActorEvents/Events/g in global config
* Manage event sending rate for SubscribeActorEvents
* Terminate SubscribeActorEvents chan when at max height
* Document future API changes
* More clarity in actor event API docs
* More post-review changes, lots of tests for SubscribeActorEvents
Use BlockDelay as the window for receiving events on the SubscribeActorEvents
channel. We expect the user to have received the initial batch of historical
events (if any) in one block's time. For real-time events we expect them to
not fall behind by roughly one block's time.
* Remove duplicate code from actor event type marshalling tests
Reduce verbosity and remove duplicate test logic from actor event types
JSON marshalling tests.
* Rename actor events test to follow go convention
Add missing `s` to `actor_events` test file to follow golang convention
used across the repo.
* Run actor events table tests in deterministic order
Refactor `map` usage for actor event table tests to ensure deterministic
test execution order, making debugging potential issues easier. If
non-determinism is a target, leverage Go's built-in parallel testing
capabilities.
* Reduce scope for filter removal failure when getting actor events
Use a fresh context to remove the temporary filter installed solely to
get the actor events. This should reduce chances of failure in a case
where the original context may be expired/cancelled.
Refactor removal into a `defer` statement for a more readable, concise
return statement.
* Use fixed RNG seed for actor event tests
Improve determinism in actor event tests by using a fixed RNG seed. This
makes up a more reproducible test suit.
* Use provided libraries to assert eventual conditions
Use the functionalities already provided by `testify` to assert eventual
conditions, and remove the use of `time.Sleep`.
Remove duplicate code in utility functions that are already defined.
Refactor assertion helper functions to use consistent terminology:
"require" implies fatal error, whereas "assert" implies error where the
test may proceed executing.
* Update changelog for actor events APIs
* Fix concerns and docs identified by review
* Update actor bundle to v13.0.0-rc3
Update actor bundle to v13.0.0-rc3
* Prep Lotus v1.26.0-rc1
- For sanity reverting the mainnet upgrade epoch to 99999999, and then only set it when cutting the final release
-Update Calibnet CIDs to v13.0.0-rc3
- Add GetActorEvents, SubscribeActorEvents, GetAllClaims and GetAllAllocations methods to the changelog
Co-Authored-By: Jiaying Wang <42981373+jennijuju@users.noreply.github.com>
* Update CHANGELOG.md
Co-authored-by: Masih H. Derkani <m@derkani.org>
* Make gen
Make gen
* fix: beacon: validate drand change at nv16 correctly
* bump to v1.26.0-rc2
* test: cleanup ddo verified itest, extract steps to functions
also add allocation-removed event case
* test: extract verified DDO test to separate file, add more checks
* test: add additional actor events checks
* Add verification for "deal-activated" actor event
* docs(drand): document the meaning of "IsChained" (#11692)
* Resolve conflicts
I encountered multiple issues when trying to run make gen. And these changes fixed a couple of them:
- go mod tidy
- Remove RaftState/RaftLeader
- Revert `if ts.Height() > claim.TermMax+claim.TermStart || !cctx.IsSet("expired")` to the what is in the release/v1.26.0: `if tsHeight > val.TermMax || !expired`
* fixup imports, make jen
* Update version
Update version in master to v1.27.0-dev
* Update node/impl/full/dummy.go
Co-authored-by: Łukasz Magiera <magik6k@users.noreply.github.com>
* Adjust ListClaimsCmd
Adjust ListClaimsCmd according to review
---------
Signed-off-by: samuelarogbonlo <sbayo971@gmail.com>
Co-authored-by: TippyFlits <james.bluett@protocol.ai>
Co-authored-by: Aayush <arajasek94@gmail.com>
Co-authored-by: Łukasz Magiera <magik6k@users.noreply.github.com>
Co-authored-by: Jennifer Wang <jiayingw703@gmail.com>
Co-authored-by: Shrenuj Bansal <shrenuj.bansal@protocol.ai>
Co-authored-by: Andrew Jackson (Ajax) <snadrus@gmail.com>
Co-authored-by: Steven Allen <steven@stebalien.com>
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Samuel Arogbonlo <47984109+samuelarogbonlo@users.noreply.github.com>
Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com>
Co-authored-by: tom123222 <160735201+tom123222@users.noreply.github.com>
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Masih H. Derkani <m@derkani.org>
Co-authored-by: Jiaying Wang <42981373+jennijuju@users.noreply.github.com>
2024-03-12 09:33:58 +00:00
{
Name : "Events" ,
Type : "EventsConfig" ,
Comment : ` ` ,
} ,
2023-03-13 10:10:36 +00:00
{
Name : "Index" ,
Type : "IndexConfig" ,
2023-08-01 15:28:47 +00:00
Comment : ` ` ,
} ,
{
Name : "FaultReporter" ,
Type : "FaultReporterConfig" ,
2023-03-13 10:10:36 +00:00
Comment : ` ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"HarmonyDB" : {
2023-07-14 23:05:49 +00:00
{
Name : "Hosts" ,
Type : "[]string" ,
Comment : ` HOSTS is a list of hostnames to nodes running YugabyteDB
in a cluster . Only 1 is required ` ,
} ,
{
Name : "Username" ,
Type : "string" ,
Comment : ` The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default. ` ,
} ,
{
Name : "Password" ,
Type : "string" ,
Comment : ` The password for the related username. Blank for default. ` ,
} ,
{
Name : "Database" ,
Type : "string" ,
Comment : ` The database (logical partition) within Yugabyte. Blank for default. ` ,
} ,
{
Name : "Port" ,
Type : "string" ,
Comment : ` The port to find Yugabyte. Blank for default. ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"IndexConfig" : {
2023-03-13 10:10:36 +00:00
{
Name : "EnableMsgIndex" ,
Type : "bool" ,
2023-06-09 16:46:27 +00:00
Comment : ` EXPERIMENTAL FEATURE . USE WITH CAUTION
EnableMsgIndex enables indexing of messages on chain . ` ,
2023-03-13 10:10:36 +00:00
} ,
2021-07-23 11:55:50 +00:00
} ,
2023-09-22 21:15:18 +00:00
"IndexProviderConfig" : {
2022-03-02 13:45:09 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Enable" ,
2022-03-02 13:45:09 +00:00
Type : "bool" ,
Comment : ` Enable set whether to enable indexing announcement to the network and expose endpoints that
2022-04-13 07:25:33 +00:00
allow indexer nodes to process announcements . Enabled by default . ` ,
2022-03-02 13:45:09 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "EntriesCacheCapacity" ,
2022-03-02 13:45:09 +00:00
Type : "int" ,
Comment : ` EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
entries . Defaults to 1024 if not specified . The cache is evicted using LRU policy . The
maximum storage used by the cache is a factor of EntriesCacheCapacity , EntriesChunkSize and
the length of multihashes being advertised . For example , advertising 128 - bit long multihashes
with the default EntriesCacheCapacity , and EntriesChunkSize means the cache size can grow to
256 MiB when full . ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "EntriesChunkSize" ,
2022-03-02 13:45:09 +00:00
Type : "int" ,
Comment : ` EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk .
Defaults to 16384 if not specified . Note that chunks are chained together for indexing
advertisements that include more multihashes than the configured EntriesChunkSize . ` ,
} ,
{
2024-03-15 21:38:13 +00:00
Name : "TopicName" ,
Type : "string" ,
2023-09-20 21:11:14 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` TopicName sets the topic name on which the changes to the advertised content are announced .
If not explicitly specified , the topic name is automatically inferred from the network name
in following format : ' / indexer / ingest / < network - name > '
Defaults to empty , which implies the topic name is inferred from network name . ` ,
2023-09-20 21:11:14 +00:00
} ,
{
2024-03-15 21:38:13 +00:00
Name : "PurgeCacheOnStart" ,
Type : "bool" ,
2023-09-20 21:11:14 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine
starts . By default , the cache is rehydrated from previously cached entries stored in
datastore if any is present . ` ,
2023-09-20 21:11:14 +00:00
} ,
2024-03-15 21:38:13 +00:00
} ,
"JournalConfig" : {
2023-09-20 21:11:14 +00:00
{
2024-03-15 21:38:13 +00:00
Name : "DisabledEvents" ,
Type : "string" ,
2023-09-20 21:11:14 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` Events of the form: "system1:event1,system1:event2[,...]" ` ,
2023-10-25 19:13:56 +00:00
} ,
2024-03-15 21:38:13 +00:00
} ,
"Libp2p" : {
2023-10-25 19:13:56 +00:00
{
2024-03-15 21:38:13 +00:00
Name : "ListenAddresses" ,
Type : "[]string" ,
2023-10-25 19:13:56 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` Binding address for the libp2p host - 0 means random port .
Format : multiaddress ; see https : //multiformats.io/multiaddr/`,
2023-10-25 19:13:56 +00:00
} ,
{
2024-03-15 21:38:13 +00:00
Name : "AnnounceAddresses" ,
Type : "[]string" ,
2023-10-25 19:13:56 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` Addresses to explicitally announce to other peers . If not specified ,
all interface addresses are announced
Format : multiaddress ` ,
2023-09-20 21:11:14 +00:00
} ,
{
2024-03-15 21:38:13 +00:00
Name : "NoAnnounceAddresses" ,
Type : "[]string" ,
2023-09-20 21:11:14 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` Addresses to not announce
Format : multiaddress ` ,
2023-09-20 21:11:14 +00:00
} ,
{
2024-03-15 21:38:13 +00:00
Name : "BootstrapPeers" ,
Type : "[]string" ,
2023-09-20 21:11:14 +00:00
Comment : ` ` ,
} ,
{
2024-03-15 21:38:13 +00:00
Name : "ProtectedPeers" ,
Type : "[]string" ,
2023-09-20 21:11:14 +00:00
Comment : ` ` ,
} ,
{
2024-03-15 21:38:13 +00:00
Name : "DisableNatPortMap" ,
Type : "bool" ,
2023-09-20 21:11:14 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` When not disabled ( default ) , lotus asks NAT devices ( e . g . , routers ) , to
open up an external port and forward it to the port lotus is running on .
When this works ( i . e . , when your router supports NAT port forwarding ) ,
it makes the local lotus node accessible from the public internet ` ,
2023-09-20 21:11:14 +00:00
} ,
{
2024-03-15 21:38:13 +00:00
Name : "ConnMgrLow" ,
Type : "uint" ,
2023-09-20 21:11:14 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` ConnMgrLow is the number of connections that the basic connection manager
will trim down to . ` ,
2023-09-20 21:11:14 +00:00
} ,
{
2024-03-15 21:38:13 +00:00
Name : "ConnMgrHigh" ,
Type : "uint" ,
2023-09-20 21:11:14 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` ConnMgrHigh is the number of connections that , when exceeded , will trigger
a connection GC operation . Note : protected / recently formed connections don ' t
count towards this limit . ` ,
2023-09-20 21:11:14 +00:00
} ,
{
2024-03-15 21:38:13 +00:00
Name : "ConnMgrGrace" ,
Type : "Duration" ,
2023-09-20 21:11:14 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` ConnMgrGrace is a time duration that new connections are immune from being
closed by the connection manager . ` ,
2023-09-20 21:11:14 +00:00
} ,
2024-03-15 21:38:13 +00:00
} ,
"Logging" : {
2023-09-20 21:11:14 +00:00
{
2024-03-15 21:38:13 +00:00
Name : "SubsystemLevels" ,
Type : "map[string]string" ,
2023-09-20 21:11:14 +00:00
2024-03-15 21:38:13 +00:00
Comment : ` SubsystemLevels specify per-subsystem log levels ` ,
2023-09-20 21:11:14 +00:00
} ,
} ,
2023-09-22 21:15:18 +00:00
"MinerAddressConfig" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "PreCommitControl" ,
2021-07-23 13:18:20 +00:00
Type : "[]string" ,
2021-07-23 13:40:30 +00:00
Comment : ` Addresses to send PreCommit messages from ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "CommitControl" ,
2021-07-23 13:18:20 +00:00
Type : "[]string" ,
2021-07-23 13:40:30 +00:00
Comment : ` Addresses to send Commit messages from ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "TerminateControl" ,
2021-07-23 13:18:20 +00:00
Type : "[]string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DealPublishControl" ,
2021-07-23 13:18:20 +00:00
Type : "[]string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DisableOwnerFallback" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` DisableOwnerFallback disables usage of the owner address for messages
sent automatically ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DisableWorkerFallback" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` DisableWorkerFallback disables usage of the worker address for messages
sent automatically , if control addresses are configured .
A control address that doesn ' t have enough funds will still be chosen
over the worker address if this flag is set . ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"MinerFeeConfig" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "MaxPreCommitGasFee" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxCommitGasFee" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxPreCommitBatchGasFee" ,
2021-07-23 13:18:20 +00:00
Type : "BatchFeeConfig" ,
2021-07-23 11:55:50 +00:00
Comment : ` maxBatchFee = maxBase + maxPerSector * nSectors ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxCommitBatchGasFee" ,
2021-07-23 13:18:20 +00:00
Type : "BatchFeeConfig" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxTerminateGasFee" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxWindowPoStGasFee" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 13:40:30 +00:00
Comment : ` WindowPoSt is a high-value operation, so the default fee should be high. ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxPublishDealsFee" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxMarketBalanceAddFee" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2022-11-29 10:25:41 +00:00
Comment : ` ` ,
} ,
{
Name : "MaximizeWindowPoStFeeCap" ,
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"MinerSubsystemConfig" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "EnableMining" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "EnableSealing" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "EnableSectorStorage" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "EnableMarkets" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
2023-08-09 00:59:21 +00:00
{
Name : "EnableSectorIndexDB" ,
Type : "bool" ,
2023-08-16 15:13:52 +00:00
Comment : ` When enabled , the sector index will reside in an external database
as opposed to the local KV store in the miner process
This is useful to allow workers to bypass the lotus miner to access sector information ` ,
2023-08-09 00:59:21 +00:00
} ,
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "SealerApiInfo" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "SectorIndexApiInfo" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
2023-11-09 17:22:08 +00:00
{
Name : "DisableWindowPoSt" ,
Type : "bool" ,
Comment : ` When window post is enabled , the miner will automatically submit window post proofs
for all sectors that are eligible for window post
IF WINDOW POST IS DISABLED , THE MINER WILL NOT SUBMIT WINDOW POST PROOFS
THIS WILL RESULT IN FAULTS AND PENALTIES IF NO OTHER MECHANISM IS RUNNING
TO SUBMIT WINDOW POST PROOFS .
Note : This option is entirely disabling the window post scheduler ,
not just the builtin PoSt computation like Proving . DisableBuiltinWindowPoSt .
This option will stop lotus - miner from performing any actions related
to window post , including scheduling , submitting proofs , and recovering
sectors . ` ,
} ,
2023-11-15 12:50:31 +00:00
{
Name : "DisableWinningPoSt" ,
Type : "bool" ,
Comment : ` When winning post is disabled , the miner process will NOT attempt to mine
blocks . This should only be set when there ' s an external process mining
blocks on behalf of the miner .
When disabled and no external block producers are configured , all potential
block rewards will be missed ! ` ,
} ,
2021-07-23 11:55:50 +00:00
} ,
2023-09-22 21:15:18 +00:00
"ProvingConfig" : {
2022-03-29 01:19:11 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "ParallelCheckLimit" ,
2022-03-29 01:19:11 +00:00
Type : "int" ,
2022-07-01 20:21:19 +00:00
Comment : ` Maximum number of sector checks to run in parallel . ( 0 = unlimited )
WARNING : Setting this value too high may make the node crash by running out of stack
WARNING : Setting this value too low may make sector challenge reading much slower , resulting in failed PoSt due
to late submission .
After changing this option , confirm that the new value works in your setup by invoking
2022-07-01 20:20:05 +00:00
' lotus - miner proving compute window - post 0 ' ` ,
2022-11-17 17:25:30 +00:00
} ,
{
Name : "SingleCheckTimeout" ,
Type : "Duration" ,
Comment : ` Maximum amount of time a proving pre - check can take for a sector . If the check times out the sector will be skipped
WARNING : Setting this value too low risks in sectors being skipped even though they are accessible , just reading the
test challenge took longer than this timeout
WARNING : Setting this value too high risks missing PoSt deadline in case IO operations related to this sector are
blocked ( e . g . in case of disconnected NFS mount ) ` ,
} ,
{
Name : "PartitionCheckTimeout" ,
Type : "Duration" ,
Comment : ` Maximum amount of time a proving pre - check can take for an entire partition . If the check times out , sectors in
the partition which didn ' t get checked on time will be skipped
WARNING : Setting this value too low risks in sectors being skipped even though they are accessible , just reading the
test challenge took longer than this timeout
WARNING : Setting this value too high risks missing PoSt deadline in case IO operations related to this partition are
blocked or slow ` ,
2022-07-01 19:24:54 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DisableBuiltinWindowPoSt" ,
2022-07-01 19:24:54 +00:00
Type : "bool" ,
2022-07-01 20:21:19 +00:00
Comment : ` Disable Window PoSt computation on the lotus - miner process even if no window PoSt workers are present .
WARNING : If no windowPoSt workers are connected , window PoSt WILL FAIL resulting in faulty sectors which will need
to be recovered . Before enabling this option , make sure your PoSt workers work correctly .
After changing this option , confirm that the new value works in your setup by invoking
2022-07-01 20:20:05 +00:00
' lotus - miner proving compute window - post 0 ' ` ,
2022-07-01 19:24:54 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DisableBuiltinWinningPoSt" ,
2022-07-01 19:24:54 +00:00
Type : "bool" ,
2022-07-01 20:21:19 +00:00
Comment : ` Disable Winning PoSt computation on the lotus - miner process even if no winning PoSt workers are present .
WARNING : If no WinningPoSt workers are connected , Winning PoSt WILL FAIL resulting in lost block rewards .
2022-07-01 19:24:54 +00:00
Before enabling this option , make sure your PoSt workers work correctly . ` ,
2022-03-29 01:19:11 +00:00
} ,
2022-07-01 20:20:05 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "DisableWDPoStPreChecks" ,
2022-07-01 20:20:05 +00:00
Type : "bool" ,
2022-07-01 20:21:19 +00:00
Comment : ` Disable WindowPoSt provable sector readability checks .
In normal operation , when preparing to compute WindowPoSt , lotus - miner will perform a round of reading challenges
from all sectors to confirm that those sectors can be proven . Challenges read in this process are discarded , as
2022-07-04 15:17:00 +00:00
we ' re only interested in checking that sector data can be read .
2022-07-01 20:21:19 +00:00
When using builtin proof computation ( no PoSt workers , and DisableBuiltinWindowPoSt is set to false ) , this process
can save a lot of time and compute resources in the case that some sectors are not readable - this is caused by
the builtin logic not skipping snark computation when some sectors need to be skipped .
When using PoSt workers , this process is mostly redundant , with PoSt workers challenges will be read once , and
if challenges for some sectors aren ' t readable , those sectors will just get skipped .
2022-07-04 15:17:00 +00:00
Disabling sector pre - checks will slightly reduce IO load when proving sectors , possibly resulting in shorter
2022-07-01 20:21:19 +00:00
time to produce window PoSt . In setups with good IO capabilities the effect of this option on proving time should
be negligible .
NOTE : It likely is a bad idea to disable sector pre - checks in setups with no PoSt workers .
NOTE : Even when this option is enabled , recovering sectors will be checked before recovery declaration message is
sent to the chain
After changing this option , confirm that the new value works in your setup by invoking
2022-07-01 20:20:05 +00:00
' lotus - miner proving compute window - post 0 ' ` ,
} ,
2022-07-07 10:33:40 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "MaxPartitionsPerPoStMessage" ,
2022-07-07 10:33:40 +00:00
Type : "int" ,
2023-10-10 20:34:45 +00:00
Comment : ` Maximum number of partitions to prove in a single SubmitWindowPoSt messace . 0 = network limit ( 3 in nv21 )
2022-07-07 10:33:40 +00:00
A single partition may contain up to 2349 32 GiB sectors , or 2300 64 GiB sectors .
2023-10-10 20:34:45 +00:00
//
2022-10-04 18:44:00 +00:00
Note that setting this value lower may result in less efficient gas use - more messages will be sent ,
2022-07-07 10:33:40 +00:00
to prove each deadline , resulting in more total gas use ( but each message will have lower gas limit )
Setting this value above the network limit has no effect ` ,
} ,
2022-07-07 14:52:22 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "MaxPartitionsPerRecoveryMessage" ,
2022-07-07 14:52:22 +00:00
Type : "int" ,
Comment : ` In some cases when submitting DeclareFaultsRecovered messages ,
there may be too many recoveries to fit in a BlockGasLimit .
In those cases it may be necessary to set this value to something low ( eg 1 ) ;
Note that setting this value lower may result in less efficient gas use - more messages will be sent than needed ,
resulting in more total gas use ( but each message will have lower gas limit ) ` ,
} ,
2022-10-04 18:44:00 +00:00
{
Name : "SingleRecoveringPartitionPerPostMessage" ,
Type : "bool" ,
2022-10-04 19:21:55 +00:00
Comment : ` Enable single partition per PoSt Message for partitions containing recovery sectors
2022-10-04 18:44:00 +00:00
In cases when submitting PoSt messages which contain recovering sectors , the default network limit may still be
too high to fit in the block gas limit . In those cases , it becomes useful to only house the single partition
2022-10-04 19:21:55 +00:00
with recovering sectors in the post message
2022-10-04 18:44:00 +00:00
Note that setting this value lower may result in less efficient gas use - more messages will be sent ,
to prove each deadline , resulting in more total gas use ( but each message will have lower gas limit ) ` ,
} ,
2022-03-29 01:19:11 +00:00
} ,
2023-09-22 21:15:18 +00:00
"Pubsub" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Bootstrapper" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 13:40:30 +00:00
Comment : ` Run the node in bootstrap-node mode ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DirectPeers" ,
2021-07-23 13:18:20 +00:00
Type : "[]string" ,
2021-07-23 13:40:30 +00:00
Comment : ` DirectPeers specifies peers with direct peering agreements . These peers are
connected outside of the mesh , with all ( valid ) message unconditionally
forwarded to them . The router will maintain open connections to these peers .
Note that the peering agreement should be reciprocal with direct peers
symmetrically configured at both ends .
Type : Array of multiaddress peerinfo strings , must include peerid ( / p2p / 12 D3K ... ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "IPColocationWhitelist" ,
2021-07-23 13:18:20 +00:00
Type : "[]string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "RemoteTracer" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
2021-09-15 12:22:48 +00:00
{
2021-09-24 11:53:31 +00:00
Name : "JsonTracer" ,
2021-09-15 12:22:48 +00:00
Type : "string" ,
2021-09-29 10:51:43 +00:00
Comment : ` Path to file that will be used to output tracer content in JSON format .
If present tracer will save data to defined file .
Format : file path ` ,
2021-09-15 12:50:27 +00:00
} ,
{
Name : "ElasticSearchTracer" ,
Type : "string" ,
2021-09-29 10:51:43 +00:00
Comment : ` Connection string for elasticsearch instance .
If present tracer will save data to elasticsearch .
Format : https : //<username>:<password>@<elasticsearch_url>:<port>/`,
} ,
{
Name : "ElasticSearchIndex" ,
Type : "string" ,
Comment : ` Name of elasticsearch index that will be used to save tracer data .
This property is used only if ElasticSearchTracer propery is set . ` ,
2021-09-24 11:53:31 +00:00
} ,
{
Name : "TracerSourceAuth" ,
Type : "string" ,
2021-09-29 10:51:43 +00:00
Comment : ` Auth token that will be passed with logs to elasticsearch - used for weighted peers score. ` ,
2021-07-23 11:55:50 +00:00
} ,
} ,
2023-09-22 21:15:18 +00:00
"RetrievalPricing" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Strategy" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Default" ,
2021-07-23 13:18:20 +00:00
Type : "*RetrievalPricingDefault" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "External" ,
2021-07-23 13:18:20 +00:00
Type : "*RetrievalPricingExternal" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"RetrievalPricingDefault" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "VerifiedDealsFreeTransfer" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` VerifiedDealsFreeTransfer configures zero fees for data transfer for a retrieval deal
of a payloadCid that belongs to a verified storage deal .
This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "default" .
default value is true ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"RetrievalPricingExternal" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Path" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 11:55:50 +00:00
Comment : ` Path of the external script that will be run to price a retrieval deal .
This parameter is ONLY applicable if the retrieval pricing policy strategy has been configured to "external" . ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"SealerConfig" : {
2022-03-29 01:19:11 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "ParallelFetchLimit" ,
2022-03-29 01:19:11 +00:00
Type : "int" ,
Comment : ` ` ,
} ,
{
2022-09-16 21:45:23 +00:00
Name : "AllowSectorDownload" ,
2022-03-29 01:19:11 +00:00
Type : "bool" ,
2022-09-06 09:06:30 +00:00
Comment : ` ` ,
2022-03-29 01:19:11 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AllowAddPiece" ,
2022-03-29 01:19:11 +00:00
Type : "bool" ,
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AllowPreCommit1" ,
2022-03-29 01:19:11 +00:00
Type : "bool" ,
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AllowPreCommit2" ,
2022-03-29 01:19:11 +00:00
Type : "bool" ,
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AllowCommit" ,
2022-03-29 01:19:11 +00:00
Type : "bool" ,
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AllowUnseal" ,
2022-03-29 01:19:11 +00:00
Type : "bool" ,
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AllowReplicaUpdate" ,
2022-03-29 01:19:11 +00:00
Type : "bool" ,
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AllowProveReplicaUpdate2" ,
2022-03-29 01:19:11 +00:00
Type : "bool" ,
Comment : ` ` ,
} ,
2022-08-03 10:54:32 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "AllowRegenSectorKey" ,
2022-03-29 01:19:11 +00:00
Type : "bool" ,
Comment : ` ` ,
} ,
2022-08-03 10:54:32 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "LocalWorkerName" ,
2022-08-03 10:54:32 +00:00
Type : "string" ,
Comment : ` LocalWorkerName specifies a custom name for the builtin worker .
If set to an empty string ( default ) os hostname will be used ` ,
} ,
2022-05-23 15:32:54 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Assigner" ,
2022-05-23 15:32:54 +00:00
Type : "string" ,
Comment : ` Assigner specifies the worker assigner to use when scheduling tasks .
"utilization" ( default ) - assign tasks to workers with lowest utilization .
"spread" - assign tasks to as many distinct workers as possible . ` ,
2022-05-23 21:53:25 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DisallowRemoteFinalize" ,
2022-05-23 21:53:25 +00:00
Type : "bool" ,
2022-05-23 23:33:56 +00:00
Comment : ` DisallowRemoteFinalize when set to true will force all Finalize tasks to
run on workers with local access to both long - term storage and the sealing
path containing the sector .
--
WARNING : Only set this if all workers have access to long - term storage
paths . If this flag is enabled , and there are workers without long - term
storage access , sectors will not be moved from them , and Finalize tasks
will appear to be stuck .
--
If you see stuck Finalize tasks after enabling this setting , check
2022-05-23 21:53:25 +00:00
' lotus - miner sealing sched - diag ' and ' lotus - miner storage find [ sector num ] ' ` ,
2022-05-23 15:32:54 +00:00
} ,
2022-03-29 01:19:11 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "ResourceFiltering" ,
2022-11-01 11:01:31 +00:00
Type : "ResourceFilteringStrategy" ,
2022-03-29 01:19:11 +00:00
Comment : ` ResourceFiltering instructs the system which resource filtering strategy
to use when evaluating tasks against this worker . An empty value defaults
to "hardware" . ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"SealingConfig" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "MaxWaitDealsSectors" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2021-07-23 13:40:30 +00:00
Comment : ` Upper bound on how many sectors can be waiting for more deals to be packed in it before it begins sealing at any given time .
If the miner is accepting multiple deals in parallel , up to MaxWaitDealsSectors of new sectors will be created .
If more than MaxWaitDealsSectors deals are accepted in parallel , only MaxWaitDealsSectors deals will be processed in parallel
Note that setting this number too high in relation to deal ingestion rate may result in poor sector packing efficiency
0 = no limit ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxSealingSectors" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2022-03-17 20:12:42 +00:00
Comment : ` Upper bound on how many sectors can be sealing+upgrading at the same time when creating new CC sectors (0 = unlimited) ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxSealingSectorsForDeals" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2022-03-17 20:12:42 +00:00
Comment : ` Upper bound on how many sectors can be sealing+upgrading at the same time when creating new sectors with deals (0 = unlimited) ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "PreferNewSectorsForDeals" ,
2022-03-17 20:12:42 +00:00
Type : "bool" ,
2022-03-17 20:27:10 +00:00
Comment : ` Prefer creating new sectors even if there are sectors Available for upgrading .
This setting combined with MaxUpgradingSectors set to a value higher than MaxSealingSectorsForDeals makes it
2022-03-17 20:12:42 +00:00
possible to use fast sector upgrades to handle high volumes of storage deals , while still using the simple sealing
flow when the volume of storage deals is lower . ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxUpgradingSectors" ,
2022-03-17 20:12:42 +00:00
Type : "uint64" ,
Comment : ` Upper bound on how many sectors can be sealing+upgrading at the same time when upgrading CC sectors with deals (0 = MaxSealingSectorsForDeals) ` ,
2021-07-23 11:55:50 +00:00
} ,
2021-07-24 01:05:50 +00:00
{
2022-09-14 10:45:22 +00:00
Name : "MinUpgradeSectorExpiration" ,
Type : "uint64" ,
Comment : ` When set to a non - zero value , minimum number of epochs until sector expiration required for sectors to be considered
for upgrades ( 0 = DealMinDuration = 180 days = 518400 epochs )
Note that if all deals waiting in the input queue have lifetimes longer than this value , upgrade sectors will be
required to have expiration of at least the soonest - ending deal ` ,
} ,
{
Name : "MinTargetUpgradeSectorExpiration" ,
Type : "uint64" ,
2023-05-23 19:34:27 +00:00
Comment : ` DEPRECATED: Target expiration is no longer used ` ,
2022-09-14 10:45:22 +00:00
} ,
2021-07-24 01:05:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "CommittedCapacitySectorLifetime" ,
2021-07-24 01:05:50 +00:00
Type : "Duration" ,
Comment : ` CommittedCapacitySectorLifetime is the duration a Committed Capacity ( CC ) sector will
live before it must be extended or converted into sector containing deals before it is
2023-10-11 15:30:30 +00:00
terminated . Value must be between 180 - 1278 days ( 1278 in nv21 , 540 before nv21 ) . ` ,
2021-07-24 01:05:50 +00:00
} ,
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "WaitDealsDelay" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 13:40:30 +00:00
Comment : ` Period of time that a newly created sector will wait for more deals to be packed in to before it starts to seal .
Sectors which are fully filled will start sealing immediately ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AlwaysKeepUnsealedCopy" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 13:40:30 +00:00
Comment : ` Whether to keep unsealed copies of deal data regardless of whether the client requested that . This lets the miner
avoid the relatively high cost of unsealing the data later , at the cost of more storage space ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "FinalizeEarly" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` Run sector finalization before submitting sector proof to the chain ` ,
} ,
2022-03-26 19:50:21 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "MakeNewSectorForDeals" ,
2022-03-26 19:50:21 +00:00
Type : "bool" ,
Comment : ` Whether new sectors are created to pack incoming deals
When this is set to false no new sectors will be created for sealing incoming deals
This is useful for forcing all deals to be assigned as snap deals to sectors marked for upgrade ` ,
} ,
2022-03-16 18:29:47 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "MakeCCSectorsAvailable" ,
2022-03-16 18:29:47 +00:00
Type : "bool" ,
Comment : ` After sealing CC sectors, make them available for upgrading with deals ` ,
} ,
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "CollateralFromMinerBalance" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` Whether to use available miner balance for sector collateral instead of sending it with each message ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AvailableBalanceBuffer" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 11:55:50 +00:00
Comment : ` Minimum available balance to keep in the miner actor before sending it with messages ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DisableCollateralFallback" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` Don't send collateral with messages even if there is no available balance in the miner actor ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxPreCommitBatch" ,
2021-07-23 13:18:20 +00:00
Type : "int" ,
2021-07-23 11:55:50 +00:00
Comment : ` maximum precommit batch size - batches will be sent immediately above this size ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "PreCommitBatchWait" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 11:55:50 +00:00
Comment : ` how long to wait before submitting a batch after crossing the minimum batch size ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "PreCommitBatchSlack" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 11:55:50 +00:00
Comment : ` time buffer for forceful batch submission before sectors/deal in batch would start expiring ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "AggregateCommits" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` enable / disable commit aggregation (takes effect after nv13) ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MinCommitBatch" ,
2021-07-23 13:18:20 +00:00
Type : "int" ,
2022-05-23 16:49:01 +00:00
Comment : ` minimum batched commit size - batches above this size will eventually be sent on a timeout ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MaxCommitBatch" ,
2021-07-23 13:18:20 +00:00
Type : "int" ,
2022-05-23 16:49:01 +00:00
Comment : ` maximum batched commit size - batches will be sent immediately above this size ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "CommitBatchWait" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 11:55:50 +00:00
Comment : ` how long to wait before submitting a batch after crossing the minimum batch size ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "CommitBatchSlack" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 11:55:50 +00:00
Comment : ` time buffer for forceful batch submission before sectors/deals in batch would start expiring ` ,
} ,
2021-09-30 14:53:12 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "BatchPreCommitAboveBaseFee" ,
2021-09-30 14:53:12 +00:00
Type : "types.FIL" ,
Comment : ` network BaseFee below which to stop doing precommit batching , instead
2023-08-08 11:04:21 +00:00
sending precommit messages to the chain individually . When the basefee is
below this threshold , precommit messages will get sent out immediately . ` ,
2021-09-30 14:53:12 +00:00
} ,
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "AggregateAboveBaseFee" ,
2021-07-23 13:18:20 +00:00
Type : "types.FIL" ,
2021-07-23 11:55:50 +00:00
Comment : ` network BaseFee below which to stop doing commit aggregation , instead
submitting proofs to the chain individually ` ,
2023-04-01 23:30:32 +00:00
} ,
{
Name : "MaxSectorProveCommitsSubmittedPerEpoch" ,
Type : "uint64" ,
Comment : ` When submitting several sector prove commit messages simultaneously , this option allows you to
stagger the number of prove commits submitted per epoch
This is done because gas estimates for ProveCommits are non deterministic and increasing as a large
number of sectors get committed within the same epoch resulting in occasionally failed msgs .
Submitting a smaller number of prove commits per epoch would reduce the possibility of failed msgs ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "TerminateBatchMax" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "TerminateBatchMin" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "TerminateBatchWait" ,
2021-07-23 13:18:20 +00:00
Type : "Duration" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
2023-06-01 04:03:52 +00:00
{
2023-07-12 11:51:50 +00:00
Name : "UseSyntheticPoRep" ,
2023-06-01 04:03:52 +00:00
Type : "bool" ,
2023-10-02 12:19:13 +00:00
Comment : ` UseSyntheticPoRep, when set to true, will reduce the amount of cache data held on disk after the completion of PreCommit 2 to 11GiB. ` ,
2023-06-01 04:03:52 +00:00
} ,
chore: Merge nv22 into master (#11699)
* [WIP] feat: Add nv22 skeleton
Addition of Network Version 22 skeleton
* update FFI
* feat: drand: refactor round verification
* feat: sealing: Support nv22 DDO features in the sealing pipeline (#11226)
* Initial work supporting DDO pieces in lotus-miner
* sealing: Update pipeline input to operate on UniversalPiece
* sealing: Update pipeline checks/sealing states to operate on UniversalPiece
* sealing: Make pipeline build with UniversalPiece
* move PieceDealInfo out of api
* make gen
* make sealing pipeline unit tests pass
* fix itest ensemble build
* don't panic in SectorsStatus with deals
* stop linter from complaining about checkPieces
* fix sector import tests
* mod tidy
* sealing: Add logic for (pre)committing DDO sectors
* sealing: state-types with method defs
* DDO non-snap pipeline works(?), DDO Itests
* DDO support in snapdeals pipeline
* make gen
* update actor bundles
* update the gst market fix
* fix: chain: use PreCommitSectorsBatch2 when setting up genesis
* some bug fixes
* integration working changes
* update actor bundles
* Make TestOnboardRawPieceSnap pass
* Appease the linter
* Make deadlines test pass with v12 actors
* Update go-state-types, abstract market DealState
* make gen
* mod tidy, lint fixes
* Fix some more tests
* Bump version in master
Bump version in master
* Make gen
Make gen
* fix sender
* fix: lotus-provider: Fix winning PoSt
* fix: sql Scan cannot write to an object
* Actually show miner-addrs in info-log
Actually show miner-addrs in lotus-provider info-log
* [WIP] feat: Add nv22 skeleton
Addition of Network Version 22 skeleton
* update FFI
* ddo is now nv22
* make gen
* temp actor bundle with ddo
* use working go-state-types
* gst with v13 market migration
* update bundle, builtin.MethodsMiner.ProveCommitSectors2 -> 3
* actually working v13 migration, v13 migration itest
* Address review
* sealing: Correct DDO snap pledge math
* itests: Mixed ddo itest
* pipeline: Fix sectorWeight
* sealing: convert market deals into PAMs in mixed sectors
* sealing: make market to ddo conversion work
* fix lint
* update gst
* Update actors and GST to lastest integ branch
* commit batcher: Update ProveCommitSectors3Params builder logic
* make gen
* use builtin-actors master
* ddo: address review
* itests: Add commd assertions to ddo tests
* make gen
* gst with fixed types
* config knobs for RequireActivationSuccess
* storage: Drop obsolete flaky tasts
---------
Co-authored-by: Jennifer Wang <jiayingw703@gmail.com>
Co-authored-by: Aayush <arajasek94@gmail.com>
Co-authored-by: Shrenuj Bansal <shrenuj.bansal@protocol.ai>
Co-authored-by: Phi <orjan.roren@gmail.com>
Co-authored-by: Andrew Jackson (Ajax) <snadrus@gmail.com>
Co-authored-by: TippyFlits <james.bluett@protocol.ai>
* feat: implement FIP-0063
* chore: deps: update to go-multiaddr v0.12.2 (#11602)
* feat: fvm: update the FVM/FFI to v4.1 (#11608) (#11612)
This:
1. Adds nv22 support.
2. Updates the message tracing format.
Co-authored-by: Steven Allen <steven@stebalien.com>
* AggregateProofType nil when doing batch updates
Use latest nv22 go-state-types version with matching update
* Update to v13.0.0-rc.2 bundle
* chore: Upgrade heights and codename
Update upgrade heights
Co-Authored-By: Steven Allen <steven@stebalien.com>
* Update epoch after nv22 DRAND switch
Update epoch after nv22 DRAND switch
* Update Mango codename to Phoneix
Make the codename for the Drand-change inline with Dragon style.
* Add UpgradePhoenixHeight to API params
* set UpgradePhoenixHeight to be one hour after Dragon
* Make gen
Make gen and UpgradePhoenixHeight in butterfly and local devnet to be in line with Calibration and Mainnet
* Update epoch heights (#11637)
Update epoch heights
* new: add forest bootstrap nodes (#11636)
Signed-off-by: samuelarogbonlo <sbayo971@gmail.com>
* Merge pull request #11491 from filecoin-project/fix/remove-decommissioned-pl-bootstrap-nodes
Remove PL operated bootstrap nodes from mainnet.pi
* feat: api: new verified registry methods to get all allocations and claims (#11631)
* new verireg methods
* update changelog and add itest
* update itest and cli
* update new method's support till v9
* remove gateway APIs
* fix cli internal var names
* chore:: backport #11609 to the feat/nv22 branch (#11644)
* feat: api: improve the correctness of Eth's trace_block (#11609)
* Improve the correctness of Eth's trace_block
- Improve encoding/decoding of parameters and return values:
- Encode "native" parameters and return values with Solidity ABI.
- Correctly decode parameters to "create" calls.
- Use the correct (ish) output for "create" calls.
- Handle all forms of "create".
- Make robust with respect to reverts:
- Use the actor ID/address from the trace instead of looking it up in
the state-tree (may not exist in the state-tree due to a revert).
- Gracefully handle failed actor/contract creation.
- Improve performance:
- We avoid looking anything up in the state-tree when translating the
trace, which should significantly improve performance.
- Improve code readability:
- Remove all "backtracking" logic.
- Use an "environment" struct to store temporary state instead of
attaching it to the trace.
- Fix random bugs:
- Fix an allocation bug in the "address" logic (need to set the
capacity before modifying the slice).
- Improved error checking/handling.
- Use correct types for `trace_block` action/results (create, call, etc.).
- And use the correct types for Result/Action structs instead of reusing the same "Call" action every time.
- Improve error messages.
* Make gen
Make gen
---------
Co-authored-by: Steven Allen <steven@stebalien.com>
* fix: add UpgradePhoenixHeight to StateGetNetworkParams (#11648)
* chore: deps: update to go-state-types v13.0.0-rc.1
* do NOT update the cache when running the real migration
* Merge pull request #11632 from hanabi1224/hm/drand-test
feat: drand quicknet: allow scheduling drand quicknet upgrade before nv22 on 2k devnet
* chore: deps: update to go-state-types v13.0.0-rc.2
chore: deps: update to go-state-types v13.0.0-rc.2
* feat: set migration config UpgradeEpoch for v13 actors upgrade
* Built-in actor events first draft
* itest for DDO non-market verified data w/ builtin actor events
* Tests for builtin actor events API
* Clean up DDO+Events tests, add lots of explainer comments
* Minor tweaks to events types
* Avoid duplicate messages when looking for receipts
* Rename internal events modules for clarity
* Adjust actor event API after review
* s/ActorEvents/Events/g in global config
* Manage event sending rate for SubscribeActorEvents
* Terminate SubscribeActorEvents chan when at max height
* Document future API changes
* More clarity in actor event API docs
* More post-review changes, lots of tests for SubscribeActorEvents
Use BlockDelay as the window for receiving events on the SubscribeActorEvents
channel. We expect the user to have received the initial batch of historical
events (if any) in one block's time. For real-time events we expect them to
not fall behind by roughly one block's time.
* Remove duplicate code from actor event type marshalling tests
Reduce verbosity and remove duplicate test logic from actor event types
JSON marshalling tests.
* Rename actor events test to follow go convention
Add missing `s` to `actor_events` test file to follow golang convention
used across the repo.
* Run actor events table tests in deterministic order
Refactor `map` usage for actor event table tests to ensure deterministic
test execution order, making debugging potential issues easier. If
non-determinism is a target, leverage Go's built-in parallel testing
capabilities.
* Reduce scope for filter removal failure when getting actor events
Use a fresh context to remove the temporary filter installed solely to
get the actor events. This should reduce chances of failure in a case
where the original context may be expired/cancelled.
Refactor removal into a `defer` statement for a more readable, concise
return statement.
* Use fixed RNG seed for actor event tests
Improve determinism in actor event tests by using a fixed RNG seed. This
makes up a more reproducible test suit.
* Use provided libraries to assert eventual conditions
Use the functionalities already provided by `testify` to assert eventual
conditions, and remove the use of `time.Sleep`.
Remove duplicate code in utility functions that are already defined.
Refactor assertion helper functions to use consistent terminology:
"require" implies fatal error, whereas "assert" implies error where the
test may proceed executing.
* Update changelog for actor events APIs
* Fix concerns and docs identified by review
* Update actor bundle to v13.0.0-rc3
Update actor bundle to v13.0.0-rc3
* Prep Lotus v1.26.0-rc1
- For sanity reverting the mainnet upgrade epoch to 99999999, and then only set it when cutting the final release
-Update Calibnet CIDs to v13.0.0-rc3
- Add GetActorEvents, SubscribeActorEvents, GetAllClaims and GetAllAllocations methods to the changelog
Co-Authored-By: Jiaying Wang <42981373+jennijuju@users.noreply.github.com>
* Update CHANGELOG.md
Co-authored-by: Masih H. Derkani <m@derkani.org>
* Make gen
Make gen
* fix: beacon: validate drand change at nv16 correctly
* bump to v1.26.0-rc2
* test: cleanup ddo verified itest, extract steps to functions
also add allocation-removed event case
* test: extract verified DDO test to separate file, add more checks
* test: add additional actor events checks
* Add verification for "deal-activated" actor event
* docs(drand): document the meaning of "IsChained" (#11692)
* Resolve conflicts
I encountered multiple issues when trying to run make gen. And these changes fixed a couple of them:
- go mod tidy
- Remove RaftState/RaftLeader
- Revert `if ts.Height() > claim.TermMax+claim.TermStart || !cctx.IsSet("expired")` to the what is in the release/v1.26.0: `if tsHeight > val.TermMax || !expired`
* fixup imports, make jen
* Update version
Update version in master to v1.27.0-dev
* Update node/impl/full/dummy.go
Co-authored-by: Łukasz Magiera <magik6k@users.noreply.github.com>
* Adjust ListClaimsCmd
Adjust ListClaimsCmd according to review
---------
Signed-off-by: samuelarogbonlo <sbayo971@gmail.com>
Co-authored-by: TippyFlits <james.bluett@protocol.ai>
Co-authored-by: Aayush <arajasek94@gmail.com>
Co-authored-by: Łukasz Magiera <magik6k@users.noreply.github.com>
Co-authored-by: Jennifer Wang <jiayingw703@gmail.com>
Co-authored-by: Shrenuj Bansal <shrenuj.bansal@protocol.ai>
Co-authored-by: Andrew Jackson (Ajax) <snadrus@gmail.com>
Co-authored-by: Steven Allen <steven@stebalien.com>
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Samuel Arogbonlo <47984109+samuelarogbonlo@users.noreply.github.com>
Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com>
Co-authored-by: tom123222 <160735201+tom123222@users.noreply.github.com>
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Masih H. Derkani <m@derkani.org>
Co-authored-by: Jiaying Wang <42981373+jennijuju@users.noreply.github.com>
2024-03-12 09:33:58 +00:00
{
Name : "RequireActivationSuccess" ,
Type : "bool" ,
Comment : ` Whether to abort if any sector activation in a batch fails (newly sealed sectors, only with ProveCommitSectors3). ` ,
} ,
{
Name : "RequireActivationSuccessUpdate" ,
Type : "bool" ,
Comment : ` Whether to abort if any piece activation notification returns a non-zero exit code (newly sealed sectors, only with ProveCommitSectors3). ` ,
} ,
{
Name : "RequireNotificationSuccess" ,
Type : "bool" ,
Comment : ` Whether to abort if any sector activation in a batch fails (updating sectors, only with ProveReplicaUpdates3). ` ,
} ,
{
Name : "RequireNotificationSuccessUpdate" ,
Type : "bool" ,
Comment : ` Whether to abort if any piece activation notification returns a non-zero exit code (updating sectors, only with ProveReplicaUpdates3). ` ,
} ,
2021-07-23 11:55:50 +00:00
} ,
2023-09-22 21:15:18 +00:00
"Splitstore" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "ColdStoreType" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-24 06:00:00 +00:00
Comment : ` ColdStoreType specifies the type of the coldstore .
2023-09-22 13:24:55 +00:00
It can be "discard" ( default ) for discarding cold blocks , "messages" to store only messages or "universal" to store all chain state . . ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "HotStoreType" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-24 06:00:00 +00:00
Comment : ` HotStoreType specifies the type of the hotstore .
Only currently supported value is "badger" . ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "MarkSetType" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-24 06:00:00 +00:00
Comment : ` MarkSetType specifies the type of the markset .
2022-02-06 10:28:21 +00:00
It can be "map" for in memory marking or "badger" ( default ) for on - disk marking . ` ,
2021-07-23 11:55:50 +00:00
} ,
{
2022-09-14 16:41:47 +00:00
Name : "HotStoreMessageRetention" ,
2021-07-23 13:18:20 +00:00
Type : "uint64" ,
2021-07-24 06:00:00 +00:00
Comment : ` HotStoreMessageRetention specifies the retention policy for messages , in finalities beyond
the compaction boundary ; default is 0. ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "HotStoreFullGCFrequency" ,
2021-07-24 06:00:00 +00:00
Type : "uint64" ,
2021-07-27 09:13:26 +00:00
Comment : ` HotStoreFullGCFrequency specifies how often to perform a full ( moving ) GC on the hotstore .
A value of 0 disables , while a value 1 will do full GC in every compaction .
2021-07-24 06:00:00 +00:00
Default is 20 ( about once a week ) . ` ,
2021-07-23 11:55:50 +00:00
} ,
2023-03-07 14:38:27 +00:00
{
Name : "HotStoreMaxSpaceTarget" ,
Type : "uint64" ,
Comment : ` HotStoreMaxSpaceTarget sets a target max disk size for the hotstore . Splitstore GC
will run moving GC if disk utilization gets within a threshold ( 150 GB ) of the target .
Splitstore GC will NOT run moving GC if the total size of the move would get
within 50 GB of the target , and instead will run a more aggressive online GC .
If both HotStoreFullGCFrequency and HotStoreMaxSpaceTarget are set then splitstore
2023-03-09 01:11:39 +00:00
GC will trigger moving GC if either configuration condition is met .
A reasonable minimum is 2 x fully GCed hotstore size + 50 G buffer .
At this minimum size moving GC happens every time , any smaller and moving GC won ' t
be able to run . In spring 2023 this minimum is ~ 550 GB . ` ,
2023-03-07 14:38:27 +00:00
} ,
2023-03-09 15:57:14 +00:00
{
Name : "HotStoreMaxSpaceThreshold" ,
Type : "uint64" ,
Comment : ` When HotStoreMaxSpaceTarget is set Moving GC will be triggered when total moving size
exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold ` ,
} ,
{
Name : "HotstoreMaxSpaceSafetyBuffer" ,
Type : "uint64" ,
Comment : ` Safety buffer to prevent moving GC from overflowing disk when HotStoreMaxSpaceTarget
is set . Moving GC will not occur when total moving size exceeds
HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer ` ,
} ,
2021-07-23 11:55:50 +00:00
} ,
2023-09-22 21:15:18 +00:00
"StorageMiner" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Subsystems" ,
2021-07-23 13:18:20 +00:00
Type : "MinerSubsystemConfig" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Dealmaking" ,
2021-07-23 13:18:20 +00:00
Type : "DealmakingConfig" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
2021-11-17 11:02:11 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "IndexProvider" ,
2022-02-03 11:51:01 +00:00
Type : "IndexProviderConfig" ,
2021-11-17 11:02:11 +00:00
Comment : ` ` ,
} ,
2022-03-29 01:19:11 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Proving" ,
2022-03-29 01:19:11 +00:00
Type : "ProvingConfig" ,
Comment : ` ` ,
} ,
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "Sealing" ,
2021-07-23 13:18:20 +00:00
Type : "SealingConfig" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Storage" ,
2022-03-29 01:19:11 +00:00
Type : "SealerConfig" ,
2021-07-23 13:18:20 +00:00
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Fees" ,
2021-07-23 13:18:20 +00:00
Type : "MinerFeeConfig" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "Addresses" ,
2021-07-23 13:18:20 +00:00
Type : "MinerAddressConfig" ,
integrate DAG store and CARv2 in deal-making (#6671)
This commit removes badger from the deal-making processes, and
moves to a new architecture with the dagstore as the cental
component on the miner-side, and CARv2s on the client-side.
Every deal that has been handed off to the sealing subsystem becomes
a shard in the dagstore. Shards are mounted via the LotusMount, which
teaches the dagstore how to load the related piece when serving
retrievals.
When the miner starts the Lotus for the first time with this patch,
we will perform a one-time migration of all active deals into the
dagstore. This is a lightweight process, and it consists simply
of registering the shards in the dagstore.
Shards are backed by the unsealed copy of the piece. This is currently
a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so
when it's time to acquire a shard to serve a retrieval, the unsealed
CARv1 is joined with its index (safeguarded by the dagstore), to form
a read-only blockstore, thus taking the place of the monolithic
badger.
Data transfers have been adjusted to interface directly with CARv2 files.
On inbound transfers (client retrievals, miner storage deals), we stream
the received data into a CARv2 ReadWrite blockstore. On outbound transfers
(client storage deals, miner retrievals), we serve the data off a CARv2
ReadOnly blockstore.
Client-side imports are managed by the refactored *imports.Manager
component (when not using IPFS integration). Just like it before, we use
the go-filestore library to avoid duplicating the data from the original
file in the resulting UnixFS DAG (concretely the leaves). However, the
target of those imports are what we call "ref-CARv2s": CARv2 files placed
under the `$LOTUS_PATH/imports` directory, containing the intermediate
nodes in full, and the leaves as positional references to the original file
on disk.
Client-side retrievals are placed into CARv2 files in the location:
`$LOTUS_PATH/retrievals`.
A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore`
subcommands have been introduced on the miner-side to inspect and manage
the dagstore.
Despite moving to a CARv2-backed system, the IPFS integration has been
respected, and it continues to be possible to make storage deals with data
held in an IPFS node, and to perform retrievals directly into an IPFS node.
NOTE: because the "staging" and "client" Badger blockstores are no longer
used, existing imports on the client will be rendered useless. On startup,
Lotus will enumerate all imports and print WARN statements on the log for
each import that needs to be reimported. These log lines contain these
messages:
- import lacks carv2 path; import will not work; please reimport
- import has missing/broken carv2; please reimport
At the end, we will print a "sanity check completed" message indicating
the count of imports found, and how many were deemed broken.
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DAGStore" ,
integrate DAG store and CARv2 in deal-making (#6671)
This commit removes badger from the deal-making processes, and
moves to a new architecture with the dagstore as the cental
component on the miner-side, and CARv2s on the client-side.
Every deal that has been handed off to the sealing subsystem becomes
a shard in the dagstore. Shards are mounted via the LotusMount, which
teaches the dagstore how to load the related piece when serving
retrievals.
When the miner starts the Lotus for the first time with this patch,
we will perform a one-time migration of all active deals into the
dagstore. This is a lightweight process, and it consists simply
of registering the shards in the dagstore.
Shards are backed by the unsealed copy of the piece. This is currently
a CARv1. However, the dagstore keeps CARv2 indices for all pieces, so
when it's time to acquire a shard to serve a retrieval, the unsealed
CARv1 is joined with its index (safeguarded by the dagstore), to form
a read-only blockstore, thus taking the place of the monolithic
badger.
Data transfers have been adjusted to interface directly with CARv2 files.
On inbound transfers (client retrievals, miner storage deals), we stream
the received data into a CARv2 ReadWrite blockstore. On outbound transfers
(client storage deals, miner retrievals), we serve the data off a CARv2
ReadOnly blockstore.
Client-side imports are managed by the refactored *imports.Manager
component (when not using IPFS integration). Just like it before, we use
the go-filestore library to avoid duplicating the data from the original
file in the resulting UnixFS DAG (concretely the leaves). However, the
target of those imports are what we call "ref-CARv2s": CARv2 files placed
under the `$LOTUS_PATH/imports` directory, containing the intermediate
nodes in full, and the leaves as positional references to the original file
on disk.
Client-side retrievals are placed into CARv2 files in the location:
`$LOTUS_PATH/retrievals`.
A new set of `Dagstore*` JSON-RPC operations and `lotus-miner dagstore`
subcommands have been introduced on the miner-side to inspect and manage
the dagstore.
Despite moving to a CARv2-backed system, the IPFS integration has been
respected, and it continues to be possible to make storage deals with data
held in an IPFS node, and to perform retrievals directly into an IPFS node.
NOTE: because the "staging" and "client" Badger blockstores are no longer
used, existing imports on the client will be rendered useless. On startup,
Lotus will enumerate all imports and print WARN statements on the log for
each import that needs to be reimported. These log lines contain these
messages:
- import lacks carv2 path; import will not work; please reimport
- import has missing/broken carv2; please reimport
At the end, we will print a "sanity check completed" message indicating
the count of imports found, and how many were deemed broken.
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>
Co-authored-by: Dirk McCormick <dirkmdev@gmail.com>
2021-08-16 22:34:32 +00:00
Type : "DAGStoreConfig" ,
2023-07-14 23:05:49 +00:00
Comment : ` ` ,
} ,
{
Name : "HarmonyDB" ,
Type : "HarmonyDB" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
2023-09-22 21:15:18 +00:00
"Wallet" : {
2021-07-23 11:55:50 +00:00
{
2022-09-14 16:41:47 +00:00
Name : "RemoteBackend" ,
2021-07-23 13:18:20 +00:00
Type : "string" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "EnableLedger" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
{
2022-09-14 16:41:47 +00:00
Name : "DisableLocal" ,
2021-07-23 13:18:20 +00:00
Type : "bool" ,
2021-07-23 11:55:50 +00:00
Comment : ` ` ,
} ,
} ,
}