Compare commits

..
Author SHA1 Message Date
Travis Person b252a3395c New calibration network genesis 2020-10-02 03:22:40 +00:00
Travis Person 351c222d4f New calibration network info 2020-10-02 01:48:46 +00:00
108 changed files with 957 additions and 4431 deletions
-66
View File
@@ -1,71 +1,5 @@
# Lotus changelog
# 0.8.1 / 2020-09-30
This optional release of Lotus introduces a new version of markets which switches to CBOR-map encodings, and allows datastore migrations. The release also introduces several improvements to the mining process, a few performance optimizations, and a battery of UX additions and enhancements.
## Changes
#### Dependencies
- Markets 0.7.0 with updated data stores (https://github.com/filecoin-project/lotus/pull/4089)
- Update ffi to code with blst fixes (https://github.com/filecoin-project/lotus/pull/3998)
#### Core Lotus
- Fix GetPower with no miner address (https://github.com/filecoin-project/lotus/pull/4049)
- Refactor: Move nonce generation out of mpool (https://github.com/filecoin-project/lotus/pull/3970)
#### Performance
- Implement caching syscalls for import-bench (https://github.com/filecoin-project/lotus/pull/3888)
- Fetch tipset blocks in parallel (https://github.com/filecoin-project/lotus/pull/4074)
- Optimize Tipset equals() (https://github.com/filecoin-project/lotus/pull/4056)
- Make state transition in validation async (https://github.com/filecoin-project/lotus/pull/3868)
#### Mining
- Add trace window post (https://github.com/filecoin-project/lotus/pull/4020)
- Use abstract types for Dont recompute post on revert (https://github.com/filecoin-project/lotus/pull/4022)
- Fix injectNulls logic in test miner (https://github.com/filecoin-project/lotus/pull/4058)
- Fix potential panic in FinalizeSector (https://github.com/filecoin-project/lotus/pull/4092)
- Don't recompute post on revert (https://github.com/filecoin-project/lotus/pull/3924)
- Fix some failed precommit handling (https://github.com/filecoin-project/lotus/pull/3445)
- Add --no-swap flag for worker (https://github.com/filecoin-project/lotus/pull/4107)
- Allow some single-thread tasks to run in parallel with PC2/C2 (https://github.com/filecoin-project/lotus/pull/4116)
#### UX
- Add an envvar to set address network version (https://github.com/filecoin-project/lotus/pull/4028)
- Add logging to chain export (https://github.com/filecoin-project/lotus/pull/4030)
- Add JSON output to state compute (https://github.com/filecoin-project/lotus/pull/4038)
- Wallet list CLI: Print balances/nonces (https://github.com/filecoin-project/lotus/pull/4088)
- Added an option to show or not show sector info for `lotus-miner info` (https://github.com/filecoin-project/lotus/pull/4003)
- Add a command to import an ipld object into the chainstore (https://github.com/filecoin-project/lotus/pull/3434)
- Improve the lotus-shed dealtracker (https://github.com/filecoin-project/lotus/pull/4051)
- Docs review and re-organization (https://github.com/filecoin-project/lotus/pull/3431)
- Fix wallet list (https://github.com/filecoin-project/lotus/pull/4104)
- Add an endpoint to validate whether a string is a well-formed address (https://github.com/filecoin-project/lotus/pull/4106)
- Add an option to set config path (https://github.com/filecoin-project/lotus/pull/4103)
- Add printf in TestWindowPost (https://github.com/filecoin-project/lotus/pull/4043)
- Improve miner sectors list UX (https://github.com/filecoin-project/lotus/pull/4108)
#### Tooling
- Move policy change to seal bench (https://github.com/filecoin-project/lotus/pull/4032)
- Add back network power to stats (https://github.com/filecoin-project/lotus/pull/4050)
- Conformance: Record and feed circulating supply (https://github.com/filecoin-project/lotus/pull/4078)
- Snapshot import progress bar, add HTTP support (https://github.com/filecoin-project/lotus/pull/4070)
- Add lotus shed util to validate a tipset (https://github.com/filecoin-project/lotus/pull/4065)
- tvx: a test vector extraction and execution tool (https://github.com/filecoin-project/lotus/pull/4064)
#### Bootstrap
- Add new bootstrappers (https://github.com/filecoin-project/lotus/pull/4007)
- Add Glif node to bootstrap peers (https://github.com/filecoin-project/lotus/pull/4004)
- Add one more node located in China (https://github.com/filecoin-project/lotus/pull/4041)
- Add ipfsmain bootstrapper (https://github.com/filecoin-project/lotus/pull/4067)
# 0.8.0 / 2020-09-26
This consensus-breaking release of Lotus introduces an upgrade to the network. The changes that break consensus are:
+8 -24
View File
@@ -192,9 +192,6 @@ type FullNode interface {
// MpoolPush pushes a signed message to mempool.
MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error)
// MpoolPushUntrusted pushes a signed message to mempool from untrusted sources.
MpoolPushUntrusted(context.Context, *types.SignedMessage) (cid.Cid, error)
// MpoolPushMessage atomically assigns a nonce, signs, and pushes a message
// to mempool.
// maxFee is only used when GasFeeCap/GasPremium fields aren't specified
@@ -393,16 +390,10 @@ type FullNode interface {
// StateCompute is a flexible command that applies the given messages on the given tipset.
// The messages are run as though the VM were at the provided height.
StateCompute(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*ComputeStateOutput, error)
// StateVerifierStatus returns the data cap for the given address.
// Returns nil if there is no entry in the data cap table for the
// address.
StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
// StateVerifiedClientStatus returns the data cap for the given address.
// Returns nil if there is no entry in the data cap table for the
// address.
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
// StateVerifiedClientStatus returns the address of the Verified Registry's root key
StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error)
// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
// can issue. It takes the deal size and verified status as parameters.
StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error)
@@ -484,12 +475,6 @@ type FullNode interface {
PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)
PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error)
PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)
// CreateBackup creates node backup onder the specified file name. The
// method requires that the lotus daemon is running with the
// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
// the path specified when calling CreateBackup is within the base path
CreateBackup(ctx context.Context, fpath string) error
}
type FileRef struct {
@@ -536,7 +521,6 @@ type DealInfo struct {
DealID abi.DealID
CreationTime time.Time
Verified bool
}
type MsgLookup struct {
@@ -812,14 +796,14 @@ type CirculatingSupply struct {
}
type MiningBaseInfo struct {
MinerPower types.BigInt
NetworkPower types.BigInt
Sectors []builtin.SectorInfo
WorkerKey address.Address
SectorSize abi.SectorSize
PrevBeaconEntry types.BeaconEntry
BeaconEntries []types.BeaconEntry
EligibleForMining bool
MinerPower types.BigInt
NetworkPower types.BigInt
Sectors []builtin.SectorInfo
WorkerKey address.Address
SectorSize abi.SectorSize
PrevBeaconEntry types.BeaconEntry
BeaconEntries []types.BeaconEntry
HasMinPower bool
}
type BlockTemplate struct {
-6
View File
@@ -101,12 +101,6 @@ type StorageMiner interface {
PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)
PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error)
PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)
// CreateBackup creates node backup onder the specified file name. The
// method requires that the lotus-miner is running with the
// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
// the path specified when calling CreateBackup is within the base path
CreateBackup(ctx context.Context, fpath string) error
}
type SealRes struct {
+1 -29
View File
@@ -122,9 +122,7 @@ type FullNodeStruct struct {
MpoolPending func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) `perm:"read"`
MpoolClear func(context.Context, bool) error `perm:"write"`
MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"`
MpoolPushUntrusted func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"`
MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"`
MpoolPushMessage func(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error) `perm:"sign"`
MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"`
MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"`
@@ -204,9 +202,7 @@ type FullNodeStruct struct {
StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"`
StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"`
StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"`
StateVerifierStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"`
StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"`
StateVerifiedRegistryRootKey func(ctx context.Context, tsk types.TipSetKey) (address.Address, error) `perm:"read"`
StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"`
StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"`
StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"`
@@ -243,8 +239,6 @@ type FullNodeStruct struct {
PaychVoucherCreate func(context.Context, address.Address, big.Int, uint64) (*api.VoucherCreateResult, error) `perm:"sign"`
PaychVoucherList func(context.Context, address.Address) ([]*paych.SignedVoucher, error) `perm:"write"`
PaychVoucherSubmit func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error) `perm:"sign"`
CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"`
}
}
@@ -325,8 +319,6 @@ type StorageMinerStruct struct {
PiecesListCidInfos func(ctx context.Context) ([]cid.Cid, error) `perm:"read"`
PiecesGetPieceInfo func(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) `perm:"read"`
PiecesGetCIDInfo func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"`
CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"`
}
}
@@ -561,10 +553,6 @@ func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessag
return c.Internal.MpoolPush(ctx, smsg)
}
func (c *FullNodeStruct) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) {
return c.Internal.MpoolPushUntrusted(ctx, smsg)
}
func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
return c.Internal.MpoolPushMessage(ctx, msg, spec)
}
@@ -905,18 +893,10 @@ func (c *FullNodeStruct) StateCompute(ctx context.Context, height abi.ChainEpoch
return c.Internal.StateCompute(ctx, height, msgs, tsk)
}
func (c *FullNodeStruct) StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) {
return c.Internal.StateVerifierStatus(ctx, addr, tsk)
}
func (c *FullNodeStruct) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) {
return c.Internal.StateVerifiedClientStatus(ctx, addr, tsk)
}
func (c *FullNodeStruct) StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) {
return c.Internal.StateVerifiedRegistryRootKey(ctx, tsk)
}
func (c *FullNodeStruct) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) {
return c.Internal.StateDealProviderCollateralBounds(ctx, size, verified, tsk)
}
@@ -1045,10 +1025,6 @@ func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Addr
return c.Internal.PaychVoucherSubmit(ctx, ch, sv, secret, proof)
}
func (c *FullNodeStruct) CreateBackup(ctx context.Context, fpath string) error {
return c.Internal.CreateBackup(ctx, fpath)
}
// StorageMinerStruct
func (c *StorageMinerStruct) ActorAddress(ctx context.Context) (address.Address, error) {
@@ -1289,10 +1265,6 @@ func (c *StorageMinerStruct) PiecesGetCIDInfo(ctx context.Context, payloadCid ci
return c.Internal.PiecesGetCIDInfo(ctx, payloadCid)
}
func (c *StorageMinerStruct) CreateBackup(ctx context.Context, fpath string) error {
return c.Internal.CreateBackup(ctx, fpath)
}
// WorkerStruct
func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {
+1 -22
View File
@@ -12,36 +12,15 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/impl"
)
func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) {
_ = os.Setenv("BELLMAN_NO_GPU", "1")
for _, height := range []abi.ChainEpoch{
1, // before
162, // while sealing
520, // after upgrade deal
5000, // after
} {
height := height // make linters happy by copying
t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) {
testCCUpgrade(t, b, blocktime, height)
})
}
}
func testCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) {
ctx := context.Background()
n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{
Network: build.ActorUpgradeNetworkVersion,
Height: upgradeHeight,
Migration: stmgr.UpgradeActorsV2,
}}))
n, sn := b(t, 1, OneMiner)
client := n[0].FullNode.(*impl.FullNodeAPI)
miner := sn[0]
+1 -4
View File
@@ -402,12 +402,9 @@ func testRetrieval(t *testing.T, ctx context.Context, client *impl.FullNodeAPI,
IsCAR: carExport,
}
updates, err := client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref)
if err != nil {
t.Fatal(err)
}
for update := range updates {
if update.Err != "" {
t.Fatalf("retrieval failed: %s", update.Err)
t.Fatalf("%v", err)
}
}
+1 -46
View File
@@ -71,7 +71,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
t.Fatal(err)
}
channelAmt := int64(7000)
channelAmt := int64(100000)
channelInfo, err := paymentCreator.PaychGet(ctx, createrAddr, receiverAddr, abi.NewTokenAmount(channelAmt))
if err != nil {
t.Fatal(err)
@@ -182,51 +182,6 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) {
t.Fatal("Timed out waiting for receiver to submit vouchers")
}
// Create a new voucher now that some vouchers have already been submitted
vouchRes, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(1000), 3)
if err != nil {
t.Fatal(err)
}
if vouchRes.Voucher == nil {
t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouchRes.Shortfall))
}
vdelta, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouchRes.Voucher, nil, abi.NewTokenAmount(1000))
if err != nil {
t.Fatal(err)
}
if !vdelta.Equals(abi.NewTokenAmount(1000)) {
t.Fatal("voucher didn't have the right amount")
}
// Create a new voucher whose value would exceed the channel balance
excessAmt := abi.NewTokenAmount(1000)
vouchRes, err = paymentCreator.PaychVoucherCreate(ctx, channel, excessAmt, 4)
if err != nil {
t.Fatal(err)
}
if vouchRes.Voucher != nil {
t.Fatal("Expected not to be able to create voucher whose value would exceed channel balance")
}
if !vouchRes.Shortfall.Equals(excessAmt) {
t.Fatal(fmt.Errorf("Expected voucher shortfall of %d, got %d", excessAmt, vouchRes.Shortfall))
}
// Add a voucher whose value would exceed the channel balance
vouch := &paych.SignedVoucher{ChannelAddr: channel, Amount: excessAmt, Lane: 4, Nonce: 1}
vb, err := vouch.SigningBytes()
if err != nil {
t.Fatal(err)
}
sig, err := paymentCreator.WalletSign(ctx, createrAddr, vb)
if err != nil {
t.Fatal(err)
}
vouch.Signature = sig
_, err = paymentReceiver.PaychVoucherAdd(ctx, channel, vouch, nil, abi.NewTokenAmount(1000))
if err == nil {
t.Fatal(fmt.Errorf("Expected shortfall error of %d", excessAmt))
}
// wait for the settlement period to pass before collecting
waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, paych0.SettleDelay)
+1 -2
View File
@@ -12,7 +12,6 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node"
)
type TestNode struct {
@@ -45,7 +44,7 @@ type StorageMiner struct {
//
// storage array defines storage nodes, numbers in the array specify full node
// index the storage node 'belongs' to
type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode)
type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner) ([]TestNode, []TestStorageNode)
type testSuite struct {
makeNodes APIBuilder
}
+12 -38
View File
@@ -15,11 +15,9 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/mock"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
bminer "github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/impl"
@@ -116,29 +114,8 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n,
}
func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) {
for _, height := range []abi.ChainEpoch{
1, // before
162, // while sealing
5000, // while proving
} {
height := height // copy to satisfy lints
t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) {
testWindowPostUpgrade(t, b, blocktime, nSectors, height)
})
}
}
func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int,
upgradeHeight abi.ChainEpoch) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{
Network: build.ActorUpgradeNetworkVersion,
Height: upgradeHeight,
Migration: stmgr.UpgradeActorsV2,
}}))
ctx := context.Background()
n, sn := b(t, 1, OneMiner)
client := n[0].FullNode.(*impl.FullNodeAPI)
miner := sn[0]
@@ -152,24 +129,17 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration,
}
build.Clock.Sleep(time.Second)
mine := true
done := make(chan struct{})
go func() {
defer close(done)
for ctx.Err() == nil {
for mine {
build.Clock.Sleep(blocktime)
if err := sn[0].MineOne(ctx, MineNext); err != nil {
if ctx.Err() != nil {
// context was canceled, ignore the error.
return
}
t.Error(err)
}
}
}()
defer func() {
cancel()
<-done
}()
pledgeSectors(t, ctx, miner, nSectors, 0, nil)
@@ -189,7 +159,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration,
head, err := client.ChainHead(ctx)
require.NoError(t, err)
if head.Height() > di.PeriodStart+di.WPoStProvingPeriod+2 {
if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 {
fmt.Printf("Now head.Height = %d\n", head.Height())
break
}
@@ -319,11 +289,12 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration,
pledgeSectors(t, ctx, miner, 1, nSectors, nil)
{
// Wait until proven.
di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
// wait a bit more
head, err := client.ChainHead(ctx)
require.NoError(t, err)
waitUntil := di.PeriodStart + di.WPoStProvingPeriod + 2
waitUntil := head.Height() + 10
fmt.Printf("End for head.Height > %d\n", waitUntil)
for {
@@ -344,4 +315,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration,
sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
require.Equal(t, nSectors+GenesisPreseals-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged
mine = false
<-done
}
+4 -12
View File
@@ -1,12 +1,4 @@
/dns4/bootstrap-0.testnet.fildev.network/tcp/1347/p2p/12D3KooWJTUBUjtzWJGWU1XSiY21CwmHaCNLNYn2E7jqHEHyZaP7
/dns4/bootstrap-1.testnet.fildev.network/tcp/1347/p2p/12D3KooW9yeKXha4hdrJKq74zEo99T8DhriQdWNoojWnnQbsgB3v
/dns4/bootstrap-2.testnet.fildev.network/tcp/1347/p2p/12D3KooWCrx8yVG9U9Kf7w8KLN3Edkj5ZKDhgCaeMqQbcQUoB6CT
/dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34
/dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T
/dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W
/dns4/lotus-bootstrap.forceup.cn/tcp/41778/p2p/12D3KooWFQsv3nRMUevZNWWsY1Wu6NUzUbawnWU5NcRhgKuJA37C
/dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz
/dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u
/dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt
/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d
/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP
/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWMEfi7K4zWWHP4dCf2iCeepUu9iNcVm6Xk8mTDq4E8rUi
/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWL4rBBKBRMr91Ej4b1WaSAaoHzPr2zBF9zJ4io8dDuHHT
/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWHTBk2eTBFf7F25Zdxue5iYzkwMRxN4sHApzTXq7siZ9g
/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWHRpygG9qTq2bP9SWVfgbD6KgbGxNZuEnpDjgRujJATV6
Binary file not shown.
+2 -5
View File
@@ -56,6 +56,8 @@ const SealRandomnessLookback = policy.SealRandomnessLookback
// Epochs
const TicketRandomnessLookback = abi.ChainEpoch(1)
const WinningPoStSectorSetLookback = abi.ChainEpoch(10)
// /////
// Address
@@ -70,10 +72,8 @@ const FilBase = uint64(2_000_000_000)
const FilAllocStorageMining = uint64(1_100_000_000)
const FilecoinPrecision = uint64(1_000_000_000_000_000_000)
const FilReserved = uint64(300_000_000)
var InitialRewardBalance *big.Int
var InitialFilReserved *big.Int
// TODO: Move other important consts here
@@ -81,9 +81,6 @@ func init() {
InitialRewardBalance = big.NewInt(int64(FilAllocStorageMining))
InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(int64(FilecoinPrecision)))
InitialFilReserved = big.NewInt(int64(FilReserved))
InitialFilReserved = InitialFilReserved.Mul(InitialFilReserved, big.NewInt(int64(FilecoinPrecision)))
if os.Getenv("LOTUS_ADDRESS_TYPE") == AddressMainnetEnvVar {
SetAddressNetwork(address.Mainnet)
}
+2 -9
View File
@@ -50,11 +50,11 @@ var (
SealRandomnessLookback = policy.SealRandomnessLookback
TicketRandomnessLookback = abi.ChainEpoch(1)
TicketRandomnessLookback = abi.ChainEpoch(1)
WinningPoStSectorSetLookback = abi.ChainEpoch(10)
FilBase uint64 = 2_000_000_000
FilAllocStorageMining uint64 = 1_400_000_000
FilReserved uint64 = 300_000_000
FilecoinPrecision uint64 = 1_000_000_000_000_000_000
@@ -63,13 +63,6 @@ var (
v = v.Mul(v, big.NewInt(int64(FilecoinPrecision)))
return v
}()
InitialFilReserved = func() *big.Int {
v := big.NewInt(int64(FilReserved))
v = v.Mul(v, big.NewInt(int64(FilecoinPrecision)))
return v
}()
// Actor consts
// TODO: Pull from actors when its made not private
MinDealDuration = abi.ChainEpoch(180 * builtin.EpochsInDay)
+7 -12
View File
@@ -5,7 +5,6 @@
package build
import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/actors/policy"
@@ -13,19 +12,17 @@ import (
)
var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
0: DrandIncentinet,
UpgradeSmokeHeight: DrandMainnet,
0: DrandMainnet,
}
const UpgradeBreezeHeight = 41280
const UpgradeBreezeHeight = -1
const BreezeGasTampingDuration = 120
const UpgradeSmokeHeight = 51000
const UpgradeSmokeHeight = -2
const UpgradeIgnitionHeight = 94000
const UpgradeIgnitionHeight = -3
// TODO: Actual epoch needs to be filled in
const UpgradeActorsV2Height = 128888
const UpgradeActorsV2Height = 500
// This signals our tentative epoch for mainnet launch. Can make it later, but not earlier.
// Miners, clients, developers, custodians all need time to prepare.
@@ -33,14 +30,12 @@ const UpgradeActorsV2Height = 128888
const UpgradeLiftoffHeight = 148888
func init() {
policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 40))
policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 30))
policy.SetSupportedProofTypes(
abi.RegisteredSealProof_StackedDrg512MiBV1,
abi.RegisteredSealProof_StackedDrg32GiBV1,
abi.RegisteredSealProof_StackedDrg64GiBV1,
)
SetAddressNetwork(address.Mainnet)
Devnet = false
}
+1 -1
View File
@@ -29,7 +29,7 @@ func buildType() string {
}
// BuildVersion is the local build version, set by build system
const BuildVersion = "0.8.1"
const BuildVersion = "0.8.0"
func UserVersion() string {
return BuildVersion + buildType() + CurrentCommit
-11
View File
@@ -1,7 +1,6 @@
package builtin
import (
"github.com/filecoin-project/go-address"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
@@ -21,7 +20,6 @@ import (
var SystemActorAddr = builtin0.SystemActorAddr
var BurntFundsActorAddr = builtin0.BurntFundsActorAddr
var ReserveAddress = makeAddress("t090")
// TODO: Why does actors have 2 different versions of this?
type SectorInfo = proof0.SectorInfo
@@ -88,12 +86,3 @@ func IsMultisigActor(c cid.Cid) bool {
func IsPaymentChannelActor(c cid.Cid) bool {
return c == builtin0.PaymentChannelActorCodeID || c == builtin2.PaymentChannelActorCodeID
}
func makeAddress(addr string) address.Address {
ret, err := address.NewFromString(addr)
if err != nil {
panic(err)
}
return ret
}
-2
View File
@@ -58,7 +58,6 @@ type State interface {
VestedFunds(abi.ChainEpoch) (abi.TokenAmount, error)
// Funds locked for various reasons.
LockedFunds() (LockedFunds, error)
FeeDebt() (abi.TokenAmount, error)
GetSector(abi.SectorNumber) (*SectorOnChainInfo, error)
FindSector(abi.SectorNumber) (*SectorLocation, error)
@@ -145,7 +144,6 @@ type MinerInfo struct {
SealProofType abi.RegisteredSealProof
SectorSize abi.SectorSize
WindowPoStPartitionSectors uint64
ConsensusFaultElapsed abi.ChainEpoch
}
type SectorExpiration struct {
-7
View File
@@ -4,8 +4,6 @@ import (
"bytes"
"errors"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-state-types/abi"
@@ -63,10 +61,6 @@ func (s *state0) LockedFunds() (LockedFunds, error) {
}, nil
}
func (s *state0) FeeDebt() (abi.TokenAmount, error) {
return big.Zero(), nil
}
func (s *state0) InitialPledge() (abi.TokenAmount, error) {
return s.State.InitialPledgeRequirement, nil
}
@@ -293,7 +287,6 @@ func (s *state0) Info() (MinerInfo, error) {
SealProofType: info.SealProofType,
SectorSize: info.SectorSize,
WindowPoStPartitionSectors: info.WindowPoStPartitionSectors,
ConsensusFaultElapsed: -1,
}
if info.PendingWorkerKey != nil {
+5 -9
View File
@@ -11,7 +11,6 @@ import (
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/actors/adt"
@@ -46,7 +45,7 @@ type partition2 struct {
}
func (s *state2) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) {
return s.GetAvailableBalance(bal)
return s.GetAvailableBalance(bal), nil
}
func (s *state2) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) {
@@ -61,10 +60,6 @@ func (s *state2) LockedFunds() (LockedFunds, error) {
}, nil
}
func (s *state2) FeeDebt() (abi.TokenAmount, error) {
return s.State.FeeDebt, nil
}
func (s *state2) InitialPledge() (abi.TokenAmount, error) {
return s.State.InitialPledge, nil
}
@@ -111,7 +106,9 @@ func (s *state2) NumLiveSectors() (uint64, error) {
// GetSectorExpiration returns the effective expiration of the given sector.
//
// If the sector does not expire early, the Early expiration field is 0.
// If the sector isn't found or has already been terminated, this method returns
// nil and no error. If the sector does not expire early, the Early expiration
// field is 0.
func (s *state2) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {
@@ -174,7 +171,7 @@ func (s *state2) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e
return nil, err
}
if out.Early == 0 && out.OnTime == 0 {
return nil, xerrors.Errorf("failed to find sector %d", num)
return nil, nil
}
return &out, nil
}
@@ -292,7 +289,6 @@ func (s *state2) Info() (MinerInfo, error) {
SealProofType: info.SealProofType,
SectorSize: info.SectorSize,
WindowPoStPartitionSectors: info.WindowPoStPartitionSectors,
ConsensusFaultElapsed: info.ConsensusFaultElapsed,
}
if info.PendingWorkerKey != nil {
+3 -3
View File
@@ -4,9 +4,9 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init"
paych2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych"
builtin2 "github.com/filecoin-project/specs-actors/actors/builtin"
init2 "github.com/filecoin-project/specs-actors/actors/builtin/init"
paych2 "github.com/filecoin-project/specs-actors/actors/builtin/paych"
"github.com/filecoin-project/lotus/chain/actors"
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
+2 -1
View File
@@ -27,9 +27,10 @@ type mockLaneState struct {
func NewMockPayChState(from address.Address,
to address.Address,
settlingAt abi.ChainEpoch,
toSend abi.TokenAmount,
lanes map[uint64]paych.LaneState,
) paych.State {
return &mockState{from: from, to: to, settlingAt: settlingAt, toSend: big.NewInt(0), lanes: lanes}
return &mockState{from, to, settlingAt, toSend, lanes}
}
// NewMockLaneState constructs a state for a payment channel lane with the set fixed values
-4
View File
@@ -27,10 +27,6 @@ type state0 struct {
store adt.Store
}
func (s *state0) RootKey() (address.Address, error) {
return s.State.RootKey, nil
}
func (s *state0) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) {
return getDataCap(s.store, actors.Version0, s.State.VerifiedClients, addr)
}
-4
View File
@@ -27,10 +27,6 @@ type state2 struct {
store adt.Store
}
func (s *state2) RootKey() (address.Address, error) {
return s.State.RootKey, nil
}
func (s *state2) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) {
return getDataCap(s.store, actors.Version2, s.State.VerifiedClients, addr)
}
@@ -39,7 +39,6 @@ func Load(store adt.Store, act *types.Actor) (State, error) {
type State interface {
cbor.Marshaler
RootKey() (address.Address, error)
VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
+1 -18
View File
@@ -4,6 +4,7 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/chain/actors"
market0 "github.com/filecoin-project/specs-actors/actors/builtin/market"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
power0 "github.com/filecoin-project/specs-actors/actors/builtin/power"
@@ -96,21 +97,3 @@ func DealProviderCollateralBounds(
panic("unsupported network version")
}
}
// Sets the challenge window and scales the proving period to match (such that
// there are always 48 challenge windows in a proving period).
func SetWPoStChallengeWindow(period abi.ChainEpoch) {
miner0.WPoStChallengeWindow = period
miner0.WPoStProvingPeriod = period * abi.ChainEpoch(miner0.WPoStPeriodDeadlines)
miner2.WPoStChallengeWindow = period
miner2.WPoStProvingPeriod = period * abi.ChainEpoch(miner2.WPoStPeriodDeadlines)
}
func GetWinningPoStSectorSetLookback(nwVer network.Version) abi.ChainEpoch {
if nwVer <= network.Version3 {
return 10
}
return ChainFinality
}
-3
View File
@@ -43,8 +43,5 @@ func TestAssumptions(t *testing.T) {
require.EqualValues(t, miner0.SupportedProofTypes, miner2.SupportedProofTypes)
require.Equal(t, miner0.PreCommitChallengeDelay, miner2.PreCommitChallengeDelay)
require.Equal(t, miner0.ChainFinality, miner2.ChainFinality)
require.Equal(t, miner0.WPoStChallengeWindow, miner2.WPoStChallengeWindow)
require.Equal(t, miner0.WPoStProvingPeriod, miner2.WPoStProvingPeriod)
require.Equal(t, miner0.WPoStPeriodDeadlines, miner2.WPoStPeriodDeadlines)
require.True(t, verifreg0.MinVerifiedDealSize.Equals(verifreg2.MinVerifiedDealSize))
}
+1 -1
View File
@@ -117,7 +117,7 @@ func (c *client) doRequest(
res, err := c.sendRequestToPeer(ctx, peer, req)
if err != nil {
if !xerrors.Is(err, network.ErrNoConn) {
log.Warnf("could not send request to peer %s: %s",
log.Warnf("could not connect to peer %s: %s",
peer.String(), err)
}
continue
+1 -2
View File
@@ -9,7 +9,6 @@ import (
"time"
"github.com/filecoin-project/specs-actors/actors/runtime/proof"
"github.com/google/uuid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
@@ -225,7 +224,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
},
VerifregRootKey: DefaultVerifregRootkeyActor,
RemainderAccount: DefaultRemainderAccountActor,
NetworkName: uuid.New().String(),
NetworkName: "",
Timestamp: uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()),
}
+6 -3
View File
@@ -6,8 +6,6 @@ import (
"encoding/json"
"fmt"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
cbor "github.com/ipfs/go-ipld-cbor"
@@ -298,9 +296,14 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge
return nil, nil, xerrors.Errorf("somehow overallocated filecoin (allocated = %s)", types.FIL(totalFilAllocated))
}
remAccKey, err := address.NewIDAddress(90)
if err != nil {
return nil, nil, err
}
template.RemainderAccount.Balance = remainingFil
if err := createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil {
if err := createMultisigAccount(ctx, bs, cst, state, remAccKey, template.RemainderAccount, keyIDs); err != nil {
return nil, nil, xerrors.Errorf("failed to set up remainder account: %w", err)
}
-4
View File
@@ -105,10 +105,6 @@ func checkFault(t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType str
return err
}
if other == bh.Cid() {
return nil
}
return xerrors.Errorf("produced block would trigger '%s' consensus fault; miner: %s; bh: %s, other: %s", faultType, bh.Miner, bh.Cid(), other)
}
-1
View File
@@ -42,7 +42,6 @@ var Costs = map[CostKey]int64{
{builtin0.StorageMinerActorCodeID, 16}: 5325185,
{builtin0.StorageMinerActorCodeID, 18}: 2328637,
{builtin0.StoragePowerActorCodeID, 2}: 23600956,
// TODO: Just reuse v0 values for now, this isn't actually used
{builtin2.InitActorCodeID, 2}: 8916753,
{builtin2.StorageMarketActorCodeID, 2}: 6955002,
{builtin2.StorageMarketActorCodeID, 4}: 245436108,
+11 -64
View File
@@ -55,7 +55,6 @@ var baseFeeLowerBoundFactor = types.NewInt(10)
var baseFeeLowerBoundFactorConservative = types.NewInt(100)
var MaxActorPendingMessages = 1000
var MaxUntrustedActorPendingMessages = 10
var MaxNonceGap = uint64(4)
@@ -196,17 +195,9 @@ func CapGasFee(msg *types.Message, maxFee abi.TokenAmount) {
msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap
}
func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict, untrusted bool) (bool, error) {
func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict bool) (bool, error) {
nextNonce := ms.nextNonce
nonceGap := false
maxNonceGap := MaxNonceGap
maxActorPendingMessages := MaxActorPendingMessages
if untrusted {
maxNonceGap = 0
maxActorPendingMessages = MaxUntrustedActorPendingMessages
}
switch {
case m.Message.Nonce == nextNonce:
nextNonce++
@@ -215,7 +206,7 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict, untrusted
nextNonce++
}
case strict && m.Message.Nonce > nextNonce+maxNonceGap:
case strict && m.Message.Nonce > nextNonce+MaxNonceGap:
return false, xerrors.Errorf("message nonce has too big a gap from expected nonce (Nonce: %d, nextNonce: %d): %w", m.Message.Nonce, nextNonce, ErrNonceGap)
case m.Message.Nonce > nextNonce:
@@ -251,7 +242,7 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict, untrusted
//ms.requiredFunds.Sub(ms.requiredFunds, exms.Message.Value.Int)
}
if !has && strict && len(ms.msgs) >= maxActorPendingMessages {
if !has && strict && len(ms.msgs) > MaxActorPendingMessages {
log.Errorf("too many pending messages from actor %s", m.Message.From)
return false, ErrTooManyPendingMessages
}
@@ -493,7 +484,7 @@ func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) {
}
mp.curTsLk.Lock()
publish, err := mp.addTs(m, mp.curTs, true, false)
publish, err := mp.addTs(m, mp.curTs, true)
if err != nil {
mp.curTsLk.Unlock()
return cid.Undef, err
@@ -560,7 +551,7 @@ func (mp *MessagePool) Add(m *types.SignedMessage) error {
mp.curTsLk.Lock()
defer mp.curTsLk.Unlock()
_, err = mp.addTs(m, mp.curTs, false, false)
_, err = mp.addTs(m, mp.curTs, false)
return err
}
@@ -628,7 +619,7 @@ func (mp *MessagePool) checkBalance(m *types.SignedMessage, curTs *types.TipSet)
return nil
}
func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local, untrusted bool) (bool, error) {
func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) {
snonce, err := mp.getStateNonce(m.Message.From, curTs)
if err != nil {
return false, xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrSoftValidationFailure)
@@ -650,7 +641,7 @@ func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local,
return false, err
}
return publish, mp.addLocked(m, !local, untrusted)
return publish, mp.addLocked(m, !local)
}
func (mp *MessagePool) addLoaded(m *types.SignedMessage) error {
@@ -685,17 +676,17 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error {
return err
}
return mp.addLocked(m, false, false)
return mp.addLocked(m, false)
}
func (mp *MessagePool) addSkipChecks(m *types.SignedMessage) error {
mp.lk.Lock()
defer mp.lk.Unlock()
return mp.addLocked(m, false, false)
return mp.addLocked(m, false)
}
func (mp *MessagePool) addLocked(m *types.SignedMessage, strict, untrusted bool) error {
func (mp *MessagePool) addLocked(m *types.SignedMessage, strict bool) error {
log.Debugf("mpooladd: %s %d", m.Message.From, m.Message.Nonce)
if m.Signature.Type == crypto.SigTypeBLS {
mp.blsSigCache.Add(m.Cid(), m.Signature)
@@ -722,7 +713,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage, strict, untrusted bool)
mp.pending[m.Message.From] = mset
}
incr, err := mset.add(m, mp, strict, untrusted)
incr, err := mset.add(m, mp, strict)
if err != nil {
log.Debug(err)
return err
@@ -802,50 +793,6 @@ func (mp *MessagePool) getStateBalance(addr address.Address, ts *types.TipSet) (
return act.Balance, nil
}
// this method is provided for the gateway to push messages.
// differences from Push:
// - strict checks are enabled
// - extra strict add checks are used when adding the messages to the msgSet
// that means: no nonce gaps, at most 10 pending messages for the actor
func (mp *MessagePool) PushUntrusted(m *types.SignedMessage) (cid.Cid, error) {
err := mp.checkMessage(m)
if err != nil {
return cid.Undef, err
}
// serialize push access to reduce lock contention
mp.addSema <- struct{}{}
defer func() {
<-mp.addSema
}()
msgb, err := m.Serialize()
if err != nil {
return cid.Undef, err
}
mp.curTsLk.Lock()
publish, err := mp.addTs(m, mp.curTs, false, true)
if err != nil {
mp.curTsLk.Unlock()
return cid.Undef, err
}
mp.curTsLk.Unlock()
mp.lk.Lock()
if err := mp.addLocal(m, msgb); err != nil {
mp.lk.Unlock()
return cid.Undef, err
}
mp.lk.Unlock()
if publish {
err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb)
}
return m.Cid(), err
}
func (mp *MessagePool) Remove(from address.Address, nonce uint64, applied bool) {
mp.lk.Lock()
defer mp.lk.Unlock()
+16 -47
View File
@@ -3,7 +3,6 @@ package messagesigner
import (
"bytes"
"context"
"sync"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/messagepool"
@@ -17,7 +16,7 @@ import (
"golang.org/x/xerrors"
)
const dsKeyActorNonce = "ActorNextNonce"
const dsKeyActorNonce = "ActorNonce"
var log = logging.Logger("messagesigner")
@@ -29,7 +28,6 @@ type mpoolAPI interface {
// when signing a message
type MessageSigner struct {
wallet *wallet.Wallet
lk sync.Mutex
mpool mpoolAPI
ds datastore.Batching
}
@@ -49,42 +47,25 @@ func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataD
// SignMessage increments the nonce for the message From address, and signs
// the message
func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message, cb func(*types.SignedMessage) error) (*types.SignedMessage, error) {
ms.lk.Lock()
defer ms.lk.Unlock()
// Get the next message nonce
func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) {
nonce, err := ms.nextNonce(msg.From)
if err != nil {
return nil, xerrors.Errorf("failed to create nonce: %w", err)
}
// Sign the message with the nonce
msg.Nonce = nonce
sig, err := ms.wallet.Sign(ctx, msg.From, msg.Cid().Bytes())
if err != nil {
return nil, xerrors.Errorf("failed to sign message: %w", err)
}
// Callback with the signed message
smsg := &types.SignedMessage{
return &types.SignedMessage{
Message: *msg,
Signature: *sig,
}
err = cb(smsg)
if err != nil {
return nil, err
}
// If the callback executed successfully, write the nonce to the datastore
if err := ms.saveNonce(msg.From, nonce); err != nil {
return nil, xerrors.Errorf("failed to save nonce: %w", err)
}
return smsg, nil
}, nil
}
// nextNonce gets the next nonce for the given address.
// nextNonce increments the nonce.
// If there is no nonce in the datastore, gets the nonce from the message pool.
func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) {
// Nonces used to be created by the mempool and we need to support nodes
@@ -96,22 +77,21 @@ func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) {
return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err)
}
// Get the next nonce for this address from the datastore
addrNonceKey := ms.dstoreKey(addr)
// Get the nonce for this address from the datastore
addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()})
dsNonceBytes, err := ms.ds.Get(addrNonceKey)
switch {
case xerrors.Is(err, datastore.ErrNotFound):
// If a nonce for this address hasn't yet been created in the
// datastore, just use the nonce from the mempool
return nonce, nil
case err != nil:
return 0, xerrors.Errorf("failed to get nonce from datastore: %w", err)
default:
// There is a nonce in the datastore, so unmarshall it
maj, dsNonce, err := cbg.CborReadHeader(bytes.NewReader(dsNonceBytes))
// There is a nonce in the datastore, so unmarshall and increment it
maj, val, err := cbg.CborReadHeader(bytes.NewReader(dsNonceBytes))
if err != nil {
return 0, xerrors.Errorf("failed to parse nonce from datastore: %w", err)
}
@@ -119,37 +99,26 @@ func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) {
return 0, xerrors.Errorf("bad cbor type parsing nonce from datastore")
}
dsNonce := val + 1
// The message pool nonce should be <= than the datastore nonce
if nonce <= dsNonce {
nonce = dsNonce
} else {
log.Warnf("mempool nonce was larger than datastore nonce (%d > %d)", nonce, dsNonce)
}
return nonce, nil
}
}
// saveNonce increments the nonce for this address and writes it to the
// datastore
func (ms *MessageSigner) saveNonce(addr address.Address, nonce uint64) error {
// Increment the nonce
nonce++
// Write the nonce to the datastore
addrNonceKey := ms.dstoreKey(addr)
// Write the nonce for this address to the datastore
buf := bytes.Buffer{}
_, err := buf.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, nonce))
_, err = buf.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, nonce))
if err != nil {
return xerrors.Errorf("failed to marshall nonce: %w", err)
return 0, xerrors.Errorf("failed to marshall nonce: %w", err)
}
err = ms.ds.Put(addrNonceKey, buf.Bytes())
if err != nil {
return xerrors.Errorf("failed to write nonce to datastore: %w", err)
return 0, xerrors.Errorf("failed to write nonce to datastore: %w", err)
}
return nil
}
func (ms *MessageSigner) dstoreKey(addr address.Address) datastore.Key {
return datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()})
return nonce, nil
}
+3 -46
View File
@@ -5,8 +5,6 @@ import (
"sync"
"testing"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/go-state-types/crypto"
@@ -60,7 +58,6 @@ func TestMessageSignerSignMessage(t *testing.T) {
msg *types.Message
mpoolNonce [1]uint64
expNonce uint64
cbErr error
}
tests := []struct {
name string
@@ -140,37 +137,6 @@ func TestMessageSignerSignMessage(t *testing.T) {
},
expNonce: 2,
}},
}, {
name: "recover from callback error",
msgs: []msgSpec{{
// No nonce yet in datastore
msg: &types.Message{
To: to1,
From: from1,
},
expNonce: 0,
}, {
// Increment nonce
msg: &types.Message{
To: to1,
From: from1,
},
expNonce: 1,
}, {
// Callback returns error
msg: &types.Message{
To: to1,
From: from1,
},
cbErr: xerrors.Errorf("err"),
}, {
// Callback successful, should increment nonce in datastore
msg: &types.Message{
To: to1,
From: from1,
},
expNonce: 2,
}},
}}
for _, tt := range tests {
tt := tt
@@ -183,18 +149,9 @@ func TestMessageSignerSignMessage(t *testing.T) {
if len(m.mpoolNonce) == 1 {
mpool.setNonce(m.msg.From, m.mpoolNonce[0])
}
merr := m.cbErr
smsg, err := ms.SignMessage(ctx, m.msg, func(message *types.SignedMessage) error {
return merr
})
if m.cbErr != nil {
require.Error(t, err)
require.Nil(t, smsg)
} else {
require.NoError(t, err)
require.Equal(t, m.expNonce, smsg.Message.Nonce)
}
smsg, err := ms.SignMessage(ctx, m.msg)
require.NoError(t, err)
require.Equal(t, m.expNonce, smsg.Message.Nonce)
}
})
}
+12 -84
View File
@@ -9,7 +9,6 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/network"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
"golang.org/x/xerrors"
@@ -34,80 +33,17 @@ import (
"github.com/filecoin-project/lotus/chain/vm"
)
type UpgradeFunc func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error)
type Upgrade struct {
Height abi.ChainEpoch
Network network.Version
Migration UpgradeFunc
}
type UpgradeSchedule []Upgrade
func DefaultUpgradeSchedule() UpgradeSchedule {
var us UpgradeSchedule
for _, u := range []Upgrade{{
Height: build.UpgradeBreezeHeight,
Network: network.Version1,
Migration: UpgradeFaucetBurnRecovery,
}, {
Height: build.UpgradeSmokeHeight,
Network: network.Version2,
Migration: nil,
}, {
Height: build.UpgradeIgnitionHeight,
Network: network.Version3,
Migration: UpgradeIgnition,
}, {
Height: build.UpgradeActorsV2Height,
Network: network.Version4,
Migration: UpgradeActorsV2,
}, {
Height: build.UpgradeLiftoffHeight,
Network: network.Version4,
Migration: UpgradeLiftoff,
}} {
if u.Height < 0 {
// upgrade disabled
continue
}
us = append(us, u)
}
return us
}
func (us UpgradeSchedule) Validate() error {
// Make sure we're not trying to upgrade to version 0.
for _, u := range us {
if u.Network <= 0 {
return xerrors.Errorf("cannot upgrade to version <= 0: %d", u.Network)
}
}
// Make sure all the upgrades make sense.
for i := 1; i < len(us); i++ {
prev := &us[i-1]
curr := &us[i]
if !(prev.Network <= curr.Network) {
return xerrors.Errorf("cannot downgrade from version %d to version %d", prev.Network, curr.Network)
}
// Make sure the heights make sense.
if prev.Height < 0 {
// Previous upgrade was disabled.
continue
}
if !(prev.Height < curr.Height) {
return xerrors.Errorf("upgrade heights must be strictly increasing: upgrade %d was at height %d, followed by upgrade %d at height %d", i-1, prev.Height, i, curr.Height)
}
}
return nil
var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error){
build.UpgradeBreezeHeight: UpgradeFaucetBurnRecovery,
build.UpgradeIgnitionHeight: UpgradeIgnition,
build.UpgradeActorsV2Height: UpgradeActorsV2,
build.UpgradeLiftoffHeight: UpgradeLiftoff,
}
func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, height abi.ChainEpoch, cb ExecCallback, ts *types.TipSet) (cid.Cid, error) {
retCid := root
var err error
f, ok := sm.stateMigrations[height]
f, ok := ForksAtHeight[height]
if ok {
retCid, err = f(ctx, sm, cb, root, ts)
if err != nil {
@@ -435,13 +371,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal
func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) {
store := sm.cs.Store(ctx)
epoch := ts.Height() - 1
if build.UpgradeLiftoffHeight <= epoch {
return cid.Undef, xerrors.Errorf("liftoff height must be beyond ignition height")
}
nst, err := nv3.MigrateStateTree(ctx, store, root, epoch)
nst, err := nv3.MigrateStateTree(ctx, store, root, build.UpgradeIgnitionHeight)
if err != nil {
return cid.Undef, xerrors.Errorf("migrating actors state: %w", err)
}
@@ -466,7 +396,7 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo
return cid.Undef, xerrors.Errorf("second split address: %w", err)
}
err = resetGenesisMsigs(ctx, sm, store, tree, build.UpgradeLiftoffHeight)
err = resetGenesisMsigs(ctx, sm, store, tree)
if err != nil {
return cid.Undef, xerrors.Errorf("resetting genesis msig start epochs: %w", err)
}
@@ -481,7 +411,7 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo
return cid.Undef, xerrors.Errorf("splitting second msig: %w", err)
}
err = nv3.CheckStateTree(ctx, store, nst, epoch, builtin0.TotalFilecoin)
err = nv3.CheckStateTree(ctx, store, nst, build.UpgradeIgnitionHeight, builtin0.TotalFilecoin)
if err != nil {
return cid.Undef, xerrors.Errorf("sanity check after ignition upgrade failed: %w", err)
}
@@ -492,14 +422,12 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo
func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) {
store := sm.cs.Store(ctx)
epoch := ts.Height() - 1
info, err := store.Put(ctx, new(types.StateInfo))
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create new state info for actors v2: %w", err)
}
newHamtRoot, err := m2.MigrateStateTree(ctx, store, root, epoch, m2.DefaultConfig())
newHamtRoot, err := m2.MigrateStateTree(ctx, store, root, build.UpgradeActorsV2Height, m2.DefaultConfig())
if err != nil {
return cid.Undef, xerrors.Errorf("upgrading to actors v2: %w", err)
}
@@ -699,7 +627,7 @@ func makeKeyAddr(splitAddr address.Address, count uint64) (address.Address, erro
return addr, nil
}
func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree, startEpoch abi.ChainEpoch) error {
func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree) error {
gb, err := sm.cs.GetGenesis()
if err != nil {
return xerrors.Errorf("getting genesis block: %w", err)
@@ -728,7 +656,7 @@ func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store,
return xerrors.Errorf("reading multisig state: %w", err)
}
currState.StartEpoch = startEpoch
currState.StartEpoch = build.UpgradeLiftoffHeight
currActor.Head, err = store.Put(ctx, &currState)
if err != nil {
+29 -34
View File
@@ -19,6 +19,7 @@ import (
lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/stmgr"
. "github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
@@ -109,57 +110,51 @@ func TestForkHeightTriggers(t *testing.T) {
t.Fatal(err)
}
sm := NewStateManager(cg.ChainStore())
inv := vm.NewActorRegistry()
// predicting the address here... may break if other assumptions change
taddr, err := address.NewIDAddress(1002)
if err != nil {
t.Fatal(err)
}
sm, err := NewStateManagerWithUpgradeSchedule(
cg.ChainStore(), UpgradeSchedule{{
Network: 1,
Height: testForkHeight,
Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback,
root cid.Cid, ts *types.TipSet) (cid.Cid, error) {
cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore())
stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) {
cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore())
st, err := sm.StateTree(root)
if err != nil {
return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
}
st, err := sm.StateTree(root)
if err != nil {
return cid.Undef, xerrors.Errorf("getting state tree: %w", err)
}
act, err := st.GetActor(taddr)
if err != nil {
return cid.Undef, err
}
act, err := st.GetActor(taddr)
if err != nil {
return cid.Undef, err
}
var tas testActorState
if err := cst.Get(ctx, act.Head, &tas); err != nil {
return cid.Undef, xerrors.Errorf("in fork handler, failed to run get: %w", err)
}
var tas testActorState
if err := cst.Get(ctx, act.Head, &tas); err != nil {
return cid.Undef, xerrors.Errorf("in fork handler, failed to run get: %w", err)
}
tas.HasUpgraded = 55
tas.HasUpgraded = 55
ns, err := cst.Put(ctx, &tas)
if err != nil {
return cid.Undef, err
}
ns, err := cst.Put(ctx, &tas)
if err != nil {
return cid.Undef, err
}
act.Head = ns
act.Head = ns
if err := st.SetActor(taddr, act); err != nil {
return cid.Undef, err
}
if err := st.SetActor(taddr, act); err != nil {
return cid.Undef, err
}
return st.Flush(ctx)
}}})
if err != nil {
t.Fatal(err)
return st.Flush(ctx)
}
inv := vm.NewActorRegistry()
inv.Register(nil, testActor{})
sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) {
nvm, err := vm.NewVM(ctx, vmopt)
if err != nil {
+43 -99
View File
@@ -39,21 +39,9 @@ import (
var log = logging.Logger("statemgr")
type versionSpec struct {
networkVersion network.Version
atOrBelow abi.ChainEpoch
}
type StateManager struct {
cs *store.ChainStore
// Determines the network version at any given epoch.
networkVersions []versionSpec
latestVersion network.Version
// Maps chain epochs to upgrade functions.
stateMigrations map[abi.ChainEpoch]UpgradeFunc
stCache map[string][]cid.Cid
compWait map[string]chan struct{}
stlk sync.Mutex
@@ -64,49 +52,12 @@ type StateManager struct {
}
func NewStateManager(cs *store.ChainStore) *StateManager {
sm, err := NewStateManagerWithUpgradeSchedule(cs, DefaultUpgradeSchedule())
if err != nil {
panic(fmt.Sprintf("default upgrade schedule is invalid: %s", err))
}
return sm
}
func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule) (*StateManager, error) {
// If we have upgrades, make sure they're in-order and make sense.
if err := us.Validate(); err != nil {
return nil, err
}
stateMigrations := make(map[abi.ChainEpoch]UpgradeFunc, len(us))
var networkVersions []versionSpec
lastVersion := network.Version0
if len(us) > 0 {
// If we have any upgrades, process them and create a version
// schedule.
for _, upgrade := range us {
if upgrade.Migration != nil {
stateMigrations[upgrade.Height] = upgrade.Migration
}
networkVersions = append(networkVersions, versionSpec{
networkVersion: lastVersion,
atOrBelow: upgrade.Height,
})
lastVersion = upgrade.Network
}
} else {
// Otherwise, go directly to the latest version.
lastVersion = build.NewestNetworkVersion
}
return &StateManager{
networkVersions: networkVersions,
latestVersion: lastVersion,
stateMigrations: stateMigrations,
newVM: vm.NewVM,
cs: cs,
stCache: make(map[string][]cid.Cid),
compWait: make(map[string]chan struct{}),
}, nil
newVM: vm.NewVM,
cs: cs,
stCache: make(map[string][]cid.Cid),
compWait: make(map[string]chan struct{}),
}
}
func cidsToKey(cids []cid.Cid) string {
@@ -259,18 +210,6 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
}
for i := parentEpoch; i < epoch; i++ {
if i > parentEpoch {
// run cron for null rounds if any
if err := runCron(); err != nil {
return cid.Undef, cid.Undef, err
}
pstate, err = vmi.Flush(ctx)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err)
}
}
// handle state forks
// XXX: The state tree
newState, err := sm.handleStateForks(ctx, pstate, i, cb, ts)
@@ -285,6 +224,18 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp
}
}
if i > parentEpoch {
// run cron for null rounds if any
if err := runCron(); err != nil {
return cid.Cid{}, cid.Cid{}, err
}
newState, err = vmi.Flush(ctx)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err)
}
}
vmi.SetBlockHeight(i + 1)
pstate = newState
}
@@ -1171,27 +1122,14 @@ func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch,
}
}
// After UpgradeActorsV2Height these funds are accounted for in GetFilReserveDisbursed
if height <= build.UpgradeActorsV2Height {
// continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch
vf = big.Add(vf, sm.preIgnitionGenInfos.genesisPledge)
// continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch
vf = big.Add(vf, sm.preIgnitionGenInfos.genesisMarketFunds)
}
// continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch
vf = big.Add(vf, sm.preIgnitionGenInfos.genesisPledge)
// continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch
vf = big.Add(vf, sm.preIgnitionGenInfos.genesisMarketFunds)
return vf, nil
}
func GetFilReserveDisbursed(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) {
ract, err := st.GetActor(builtin.ReserveAddress)
if err != nil {
return big.Zero(), xerrors.Errorf("failed to get reserve actor: %w", err)
}
// If money enters the reserve actor, this could lead to a negative term
return big.Sub(big.NewFromGo(build.InitialFilReserved), ract.Balance), nil
}
func GetFilMined(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) {
ractor, err := st.GetActor(reward.Address)
if err != nil {
@@ -1279,14 +1217,6 @@ func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height
return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filVested: %w", err)
}
filReserveDisbursed := big.Zero()
if height > build.UpgradeActorsV2Height {
filReserveDisbursed, err = GetFilReserveDisbursed(ctx, st)
if err != nil {
return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filReserveDisbursed: %w", err)
}
}
filMined, err := GetFilMined(ctx, st)
if err != nil {
return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filMined: %w", err)
@@ -1303,7 +1233,6 @@ func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height
}
ret := types.BigAdd(filVested, filMined)
ret = types.BigAdd(ret, filReserveDisbursed)
ret = types.BigSub(ret, filBurnt)
ret = types.BigSub(ret, filLocked)
@@ -1330,14 +1259,29 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
}
func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version {
// The epochs here are the _last_ epoch for every version, or -1 if the
// version is disabled.
for _, spec := range sm.networkVersions {
if height <= spec.atOrBelow {
return spec.networkVersion
}
// TODO: move hard fork epoch checks to a schedule defined in build/
if build.UseNewestNetwork() {
return build.NewestNetworkVersion
}
return sm.latestVersion
if height <= build.UpgradeBreezeHeight {
return network.Version0
}
if height <= build.UpgradeSmokeHeight {
return network.Version1
}
if height <= build.UpgradeIgnitionHeight {
return network.Version2
}
if height <= build.UpgradeActorsV2Height {
return network.Version3
}
return build.NewestNetworkVersion
}
func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) {
+13 -88
View File
@@ -9,11 +9,6 @@ import (
"runtime"
"strings"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/chain/actors/policy"
cid "github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
@@ -30,6 +25,7 @@ import (
exported2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
@@ -413,9 +409,8 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, error) {
var lbr abi.ChainEpoch
lb := policy.GetWinningPoStSectorSetLookback(sm.GetNtwkVersion(ctx, round))
if round > lb {
lbr = round - lb
if round > build.WinningPoStSectorSetLookback {
lbr = round - build.WinningPoStSectorSetLookback
}
// more null blocks than our lookback
@@ -495,7 +490,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule
return nil, nil
}
mpow, tpow, _, err := GetPowerRaw(ctx, sm, lbst, maddr)
mpow, tpow, hmp, err := GetPowerRaw(ctx, sm, lbst, maddr)
if err != nil {
return nil, xerrors.Errorf("failed to get power: %w", err)
}
@@ -510,21 +505,15 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule
return nil, xerrors.Errorf("resolving worker address: %w", err)
}
// TODO: Not ideal performance...This method reloads miner and power state (already looked up here and in GetPowerRaw)
eligible, err := MinerEligibleToMine(ctx, sm, maddr, ts, lbts)
if err != nil {
return nil, xerrors.Errorf("determining miner eligibility: %w", err)
}
return &api.MiningBaseInfo{
MinerPower: mpow.QualityAdjPower,
NetworkPower: tpow.QualityAdjPower,
Sectors: sectors,
WorkerKey: worker,
SectorSize: info.SectorSize,
PrevBeaconEntry: *prev,
BeaconEntries: entries,
EligibleForMining: eligible,
MinerPower: mpow.QualityAdjPower,
NetworkPower: tpow.QualityAdjPower,
Sectors: sectors,
WorkerKey: worker,
SectorSize: info.SectorSize,
PrevBeaconEntry: *prev,
BeaconEntries: entries,
HasMinPower: hmp,
}, nil
}
@@ -606,7 +595,7 @@ func GetReturnType(ctx context.Context, sm *StateManager, to address.Address, me
return reflect.New(m.Ret.Elem()).Interface().(cbg.CBORUnmarshaler), nil
}
func minerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) {
func MinerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) {
pact, err := sm.LoadActor(ctx, power.Address, ts)
if err != nil {
return false, xerrors.Errorf("loading power actor state: %w", err)
@@ -620,70 +609,6 @@ func minerHasMinPower(ctx context.Context, sm *StateManager, addr address.Addres
return ps.MinerNominalPowerMeetsConsensusMinimum(addr)
}
func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Address, baseTs *types.TipSet, lookbackTs *types.TipSet) (bool, error) {
hmp, err := minerHasMinPower(ctx, sm, addr, lookbackTs)
// TODO: We're blurring the lines between a "runtime network version" and a "Lotus upgrade epoch", is that unavoidable?
if sm.GetNtwkVersion(ctx, baseTs.Height()) <= network.Version3 {
return hmp, err
}
if err != nil {
return false, err
}
if !hmp {
return false, nil
}
// Post actors v2, also check MinerEligibleForElection with base ts
pact, err := sm.LoadActor(ctx, power.Address, baseTs)
if err != nil {
return false, xerrors.Errorf("loading power actor state: %w", err)
}
pstate, err := power.Load(sm.cs.Store(ctx), pact)
if err != nil {
return false, err
}
mact, err := sm.LoadActor(ctx, addr, baseTs)
if err != nil {
return false, xerrors.Errorf("loading miner actor state: %w", err)
}
mstate, err := miner.Load(sm.cs.Store(ctx), mact)
if err != nil {
return false, err
}
// Non-empty power claim.
if claim, found, err := pstate.MinerPower(addr); err != nil {
return false, err
} else if !found {
return false, err
} else if claim.QualityAdjPower.LessThanEqual(big.Zero()) {
return false, err
}
// No fee debt.
if debt, err := mstate.FeeDebt(); err != nil {
return false, err
} else if !debt.IsZero() {
return false, err
}
// No active consensus faults.
if mInfo, err := mstate.Info(); err != nil {
return false, err
} else if baseTs.Height() <= mInfo.ConsensusFaultElapsed {
return false, nil
}
return true, nil
}
func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi.TokenAmount, error) {
str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState())
if err != nil {
+1 -1
View File
@@ -46,7 +46,7 @@ func ComputeNextBaseFee(baseFee types.BigInt, gasLimitUsed int64, noOfBlocks int
}
func (cs *ChainStore) ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) {
if build.UpgradeBreezeHeight >= 0 && ts.Height() > build.UpgradeBreezeHeight && ts.Height() < build.UpgradeBreezeHeight+build.BreezeGasTampingDuration {
if ts.Height() > build.UpgradeBreezeHeight && ts.Height() < build.UpgradeBreezeHeight+build.BreezeGasTampingDuration {
return abi.NewTokenAmount(100), nil
}
+12 -15
View File
@@ -11,27 +11,24 @@ import (
func TestBaseFee(t *testing.T) {
tests := []struct {
basefee uint64
limitUsed int64
noOfBlocks int
preSmoke, postSmoke uint64
basefee uint64
limitUsed int64
noOfBlocks int
output uint64
}{
{100e6, 0, 1, 87.5e6, 87.5e6},
{100e6, 0, 5, 87.5e6, 87.5e6},
{100e6, build.BlockGasTarget, 1, 103.125e6, 100e6},
{100e6, build.BlockGasTarget * 2, 2, 103.125e6, 100e6},
{100e6, build.BlockGasLimit * 2, 2, 112.5e6, 112.5e6},
{100e6, build.BlockGasLimit * 1.5, 2, 110937500, 106.250e6},
{100e6, 0, 1, 87.5e6},
{100e6, 0, 5, 87.5e6},
{100e6, build.BlockGasTarget, 1, 103.125e6},
{100e6, build.BlockGasTarget * 2, 2, 103.125e6},
{100e6, build.BlockGasLimit * 2, 2, 112.5e6},
{100e6, build.BlockGasLimit * 1.5, 2, 110937500},
}
for _, test := range tests {
test := test
t.Run(fmt.Sprintf("%v", test), func(t *testing.T) {
preSmoke := ComputeNextBaseFee(types.NewInt(test.basefee), test.limitUsed, test.noOfBlocks, build.UpgradeSmokeHeight-1)
assert.Equal(t, fmt.Sprintf("%d", test.preSmoke), preSmoke.String())
postSmoke := ComputeNextBaseFee(types.NewInt(test.basefee), test.limitUsed, test.noOfBlocks, build.UpgradeSmokeHeight+1)
assert.Equal(t, fmt.Sprintf("%d", test.postSmoke), postSmoke.String())
output := ComputeNextBaseFee(types.NewInt(test.basefee), test.limitUsed, test.noOfBlocks, 0)
assert.Equal(t, fmt.Sprintf("%d", test.output), output.String())
})
}
}
+17 -29
View File
@@ -1195,6 +1195,13 @@ func recurseLinks(bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid.
}
func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, w io.Writer) error {
if ts == nil {
ts = cs.GetHeaviestTipSet()
}
seen := cid.NewSet()
walked := cid.NewSet()
h := &car.CarHeader{
Roots: ts.Cids(),
Version: 1,
@@ -1204,28 +1211,6 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
return xerrors.Errorf("failed to write car header: %s", err)
}
return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, func(c cid.Cid) error {
blk, err := cs.bs.Get(c)
if err != nil {
return xerrors.Errorf("writing object to car, bs.Get: %w", err)
}
if err := carutil.LdWrite(w, c.Bytes(), blk.RawData()); err != nil {
return xerrors.Errorf("failed to write block to car output: %w", err)
}
return nil
})
}
func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, cb func(cid.Cid) error) error {
if ts == nil {
ts = cs.GetHeaviestTipSet()
}
seen := cid.NewSet()
walked := cid.NewSet()
blocksToWalk := ts.Cids()
currentMinHeight := ts.Height()
@@ -1234,15 +1219,15 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
return nil
}
if err := cb(blk); err != nil {
return err
}
data, err := cs.bs.Get(blk)
if err != nil {
return xerrors.Errorf("getting block: %w", err)
}
if err := carutil.LdWrite(w, blk.Bytes(), data.RawData()); err != nil {
return xerrors.Errorf("failed to write block to car output: %w", err)
}
var b types.BlockHeader
if err := b.UnmarshalCBOR(bytes.NewBuffer(data.RawData())); err != nil {
return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err)
@@ -1289,11 +1274,14 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
if c.Prefix().Codec != cid.DagCBOR {
continue
}
if err := cb(c); err != nil {
return err
data, err := cs.bs.Get(c)
if err != nil {
return xerrors.Errorf("writing object to car (get %s): %w", c, err)
}
if err := carutil.LdWrite(w, c.Bytes(), data.RawData()); err != nil {
return xerrors.Errorf("failed to write out car object: %w", err)
}
}
}
+11 -21
View File
@@ -44,11 +44,7 @@ var log = logging.Logger("sub")
var ErrSoftFailure = errors.New("soft validation failure")
var ErrInsufficientPower = errors.New("incoming block's miner does not have minimum power")
func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bs bserv.BlockService, cmgr connmgr.ConnManager) {
// Timeout after (block time + propagation delay). This is useless at
// this point.
timeout := time.Duration(build.BlockDelaySecs+build.PropagationDelaySecs) * time.Second
func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bserv bserv.BlockService, cmgr connmgr.ConnManager) {
for {
msg, err := bsub.Next(ctx)
if err != nil {
@@ -69,22 +65,15 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
src := msg.GetFrom()
go func() {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
// NOTE: we could also share a single session between
// all requests but that may have other consequences.
ses := bserv.NewSession(ctx, bs)
start := build.Clock.Now()
log.Debug("about to fetch messages for block from pubsub")
bmsgs, err := FetchMessagesByCids(ctx, ses, blk.BlsMessages)
bmsgs, err := FetchMessagesByCids(context.TODO(), bserv, blk.BlsMessages)
if err != nil {
log.Errorf("failed to fetch all bls messages for block received over pubusb: %s; source: %s", err, src)
return
}
smsgs, err := FetchSignedMessagesByCids(ctx, ses, blk.SecpkMessages)
smsgs, err := FetchSignedMessagesByCids(context.TODO(), bserv, blk.SecpkMessages)
if err != nil {
log.Errorf("failed to fetch all secpk messages for block received over pubusb: %s; source: %s", err, src)
return
@@ -109,7 +98,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
func FetchMessagesByCids(
ctx context.Context,
bserv bserv.BlockGetter,
bserv bserv.BlockService,
cids []cid.Cid,
) ([]*types.Message, error) {
out := make([]*types.Message, len(cids))
@@ -138,7 +127,7 @@ func FetchMessagesByCids(
// FIXME: Duplicate of above.
func FetchSignedMessagesByCids(
ctx context.Context,
bserv bserv.BlockGetter,
bserv bserv.BlockService,
cids []cid.Cid,
) ([]*types.SignedMessage, error) {
out := make([]*types.SignedMessage, len(cids))
@@ -168,11 +157,12 @@ func FetchSignedMessagesByCids(
// blocks we did not request.
func fetchCids(
ctx context.Context,
bserv bserv.BlockGetter,
bserv bserv.BlockService,
cids []cid.Cid,
cb func(int, blocks.Block) error,
) error {
fetchedBlocks := bserv.GetBlocks(ctx, cids)
// FIXME: Why don't we use the context here?
fetchedBlocks := bserv.GetBlocks(context.TODO(), cids)
cidIndex := make(map[cid.Cid]int)
for i, c := range cids {
@@ -485,14 +475,14 @@ func (bv *BlockValidator) checkPowerAndGetWorkerKey(ctx context.Context, bh *typ
return address.Undef, ErrSoftFailure
}
eligible, err := stmgr.MinerEligibleToMine(ctx, bv.stmgr, bh.Miner, baseTs, lbts)
hmp, err := stmgr.MinerHasMinPower(ctx, bv.stmgr, bh.Miner, lbts)
if err != nil {
log.Warnf("failed to determine if incoming block's miner has minimum power: %s", err)
return address.Undef, ErrSoftFailure
}
if !eligible {
log.Warnf("incoming block's miner is ineligible")
if !hmp {
log.Warnf("incoming block's miner does not have minimum power")
return address.Undef, ErrInsufficientPower
}
+3 -3
View File
@@ -827,13 +827,13 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, use
return xerrors.Errorf("block is not claiming to be a winner")
}
eligible, err := stmgr.MinerEligibleToMine(ctx, syncer.sm, h.Miner, baseTs, lbts)
hp, err := stmgr.MinerHasMinPower(ctx, syncer.sm, h.Miner, lbts)
if err != nil {
return xerrors.Errorf("determining if miner has min power failed: %w", err)
}
if !eligible {
return xerrors.New("block's miner is ineligible to mine")
if !hp {
return xerrors.New("block's miner does not meet minimum power threshold")
}
rBeacon := *prevBeacon
+9 -9
View File
@@ -31,41 +31,41 @@ func init() {
var EmptyObjectCid cid.Cid
// TryCreateAccountActor creates account actors from only BLS/SECP256K1 addresses.
func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, address.Address, aerrors.ActorError) {
func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, aerrors.ActorError) {
if err := rt.chargeGasSafe(PricelistByEpoch(rt.height).OnCreateActor()); err != nil {
return nil, address.Undef, err
return nil, err
}
addrID, err := rt.state.RegisterNewAddress(addr)
if err != nil {
return nil, address.Undef, aerrors.Escalate(err, "registering actor address")
return nil, aerrors.Escalate(err, "registering actor address")
}
act, aerr := makeActor(actors.VersionForNetwork(rt.NetworkVersion()), addr)
if aerr != nil {
return nil, address.Undef, aerr
return nil, aerr
}
if err := rt.state.SetActor(addrID, act); err != nil {
return nil, address.Undef, aerrors.Escalate(err, "creating new actor failed")
return nil, aerrors.Escalate(err, "creating new actor failed")
}
p, err := actors.SerializeParams(&addr)
if err != nil {
return nil, address.Undef, aerrors.Escalate(err, "couldn't serialize params for actor construction")
return nil, aerrors.Escalate(err, "couldn't serialize params for actor construction")
}
// call constructor on account
_, aerr = rt.internalSend(builtin0.SystemActorAddr, addrID, builtin0.MethodsAccount.Constructor, big.Zero(), p)
if aerr != nil {
return nil, address.Undef, aerrors.Wrap(aerr, "failed to invoke account constructor")
return nil, aerrors.Wrap(aerr, "failed to invoke account constructor")
}
act, err = rt.state.GetActor(addrID)
if err != nil {
return nil, address.Undef, aerrors.Escalate(err, "loading newly created actor failed")
return nil, aerrors.Escalate(err, "loading newly created actor failed")
}
return act, addrID, nil
return act, nil
}
func makeActor(ver actors.Version, addr address.Address) (*types.Actor, aerrors.ActorError) {
+2 -30
View File
@@ -15,7 +15,6 @@ import (
"github.com/filecoin-project/go-state-types/network"
rtt "github.com/filecoin-project/go-state-types/rt"
rt0 "github.com/filecoin-project/specs-actors/actors/runtime"
rt2 "github.com/filecoin-project/specs-actors/v2/actors/runtime"
"github.com/ipfs/go-cid"
ipldcbor "github.com/ipfs/go-ipld-cbor"
"go.opencensus.io/trace"
@@ -27,30 +26,8 @@ import (
"github.com/filecoin-project/lotus/chain/types"
)
type Message struct {
msg types.Message
}
func (m *Message) Caller() address.Address {
if m.msg.From.Protocol() != address.ID {
panic("runtime message has a non-ID caller")
}
return m.msg.From
}
func (m *Message) Receiver() address.Address {
if m.msg.To != address.Undef && m.msg.To.Protocol() != address.ID {
panic("runtime message has a non-ID receiver")
}
return m.msg.To
}
func (m *Message) ValueReceived() abi.TokenAmount {
return m.msg.Value
}
type Runtime struct {
rt0.Message
types.Message
rt0.Syscalls
ctx context.Context
@@ -131,7 +108,6 @@ func (rt *Runtime) StorePut(x cbor.Marshaler) cid.Cid {
}
var _ rt0.Runtime = (*Runtime)(nil)
var _ rt2.Runtime = (*Runtime)(nil)
func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.ActorError) {
defer func() {
@@ -144,11 +120,7 @@ func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act
//log.Desugar().WithOptions(zap.AddStacktrace(zapcore.ErrorLevel)).
//Sugar().Errorf("spec actors failure: %s", r)
log.Errorf("spec actors failure: %s", r)
if rt.NetworkVersion() <= network.Version3 {
aerr = aerrors.Newf(1, "spec actors failure: %s", r)
} else {
aerr = aerrors.Newf(exitcode.SysErrReserved1, "spec actors failure: %s", r)
}
aerr = aerrors.Newf(1, "spec actors failure: %s", r)
}
}()
+2 -23
View File
@@ -131,15 +131,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres
rt.Abortf(exitcode.SysErrInvalidReceiver, "resolve msg.From address failed")
}
vmm.From = resF
if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version3 {
rt.Message = &vmm
} else {
resT, _ := rt.ResolveAddress(msg.To)
// may be set to undef if recipient doesn't exist yet
vmm.To = resT
rt.Message = &Message{msg: vmm}
}
rt.Message = vmm
return rt
}
@@ -265,24 +257,11 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
toActor, err := st.GetActor(msg.To)
if err != nil {
if xerrors.Is(err, types.ErrActorNotFound) {
a, aid, err := TryCreateAccountActor(rt, msg.To)
a, err := TryCreateAccountActor(rt, msg.To)
if err != nil {
return nil, aerrors.Wrapf(err, "could not create account")
}
toActor = a
if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version3 {
// Leave the rt.Message as is
} else {
nmsg := Message{
msg: types.Message{
To: aid,
From: rt.Message.Caller(),
Value: rt.Message.ValueReceived(),
},
}
rt.Message = &nmsg
}
} else {
return nil, aerrors.Escalate(err, "getting actor")
}
+1 -47
View File
@@ -81,7 +81,7 @@ func (w *Wallet) findKey(addr address.Address) (*Key, error) {
return nil, nil
}
ki, err := w.tryFind(addr)
ki, err := w.keystore.Get(KNamePrefix + addr.String())
if err != nil {
if xerrors.Is(err, types.ErrKeyInfoNotFound) {
return nil, nil
@@ -96,42 +96,6 @@ func (w *Wallet) findKey(addr address.Address) (*Key, error) {
return k, nil
}
func (w *Wallet) tryFind(addr address.Address) (types.KeyInfo, error) {
ki, err := w.keystore.Get(KNamePrefix + addr.String())
if err == nil {
return ki, err
}
if !xerrors.Is(err, types.ErrKeyInfoNotFound) {
return types.KeyInfo{}, err
}
// We got an ErrKeyInfoNotFound error
// Try again, this time with the testnet prefix
aChars := []rune(addr.String())
prefixRunes := []rune(address.TestnetPrefix)
if len(prefixRunes) != 1 {
return types.KeyInfo{}, xerrors.Errorf("unexpected prefix length: %d", len(prefixRunes))
}
aChars[0] = prefixRunes[0]
ki, err = w.keystore.Get(KNamePrefix + string(aChars))
if err != nil {
return types.KeyInfo{}, err
}
// We found it with the testnet prefix
// Add this KeyInfo with the mainnet prefix address string
err = w.keystore.Put(KNamePrefix+addr.String(), ki)
if err != nil {
return types.KeyInfo{}, err
}
return ki, nil
}
func (w *Wallet) Export(addr address.Address) (*types.KeyInfo, error) {
k, err := w.findKey(addr)
if err != nil {
@@ -165,7 +129,6 @@ func (w *Wallet) ListAddrs() ([]address.Address, error) {
sort.Strings(all)
seen := map[address.Address]struct{}{}
out := make([]address.Address, 0, len(all))
for _, a := range all {
if strings.HasPrefix(a, KNamePrefix) {
@@ -174,19 +137,10 @@ func (w *Wallet) ListAddrs() ([]address.Address, error) {
if err != nil {
return nil, xerrors.Errorf("converting name to address: %w", err)
}
if _, ok := seen[addr]; ok {
continue // got duplicate with a different prefix
}
seen[addr] = struct{}{}
out = append(out, addr)
}
}
sort.Slice(out, func(i, j int) bool {
return out[i].String() < out[j].String()
})
return out, nil
}
-125
View File
@@ -1,125 +0,0 @@
package cli
import (
"context"
"fmt"
"os"
logging "github.com/ipfs/go-log/v2"
"github.com/mitchellh/go-homedir"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/lotus/lib/backupds"
"github.com/filecoin-project/lotus/node/repo"
)
type BackupAPI interface {
CreateBackup(ctx context.Context, fpath string) error
}
type BackupApiFn func(ctx *cli.Context) (BackupAPI, jsonrpc.ClientCloser, error)
func BackupCmd(repoFlag string, rt repo.RepoType, getApi BackupApiFn) *cli.Command {
var offlineBackup = func(cctx *cli.Context) error {
logging.SetLogLevel("badger", "ERROR") // nolint:errcheck
repoPath := cctx.String(repoFlag)
r, err := repo.NewFS(repoPath)
if err != nil {
return err
}
ok, err := r.Exists()
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("repo at '%s' is not initialized", cctx.String(repoFlag))
}
lr, err := r.LockRO(rt)
if err != nil {
return xerrors.Errorf("locking repo: %w", err)
}
defer lr.Close() // nolint:errcheck
mds, err := lr.Datastore("/metadata")
if err != nil {
return xerrors.Errorf("getting metadata datastore: %w", err)
}
bds := backupds.Wrap(mds)
fpath, err := homedir.Expand(cctx.Args().First())
if err != nil {
return xerrors.Errorf("expanding file path: %w", err)
}
out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return xerrors.Errorf("opening backup file %s: %w", fpath, err)
}
if err := bds.Backup(out); err != nil {
if cerr := out.Close(); cerr != nil {
log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err)
}
return xerrors.Errorf("backup error: %w", err)
}
if err := out.Close(); err != nil {
return xerrors.Errorf("closing backup file: %w", err)
}
return nil
}
var onlineBackup = func(cctx *cli.Context) error {
api, closer, err := getApi(cctx)
if err != nil {
return xerrors.Errorf("getting api: %w (if the node isn't running you can use the --offline flag)", err)
}
defer closer()
err = api.CreateBackup(ReqContext(cctx), cctx.Args().First())
if err != nil {
return err
}
fmt.Println("Success")
return nil
}
return &cli.Command{
Name: "backup",
Usage: "Create node metadata backup",
Description: `The backup command writes a copy of node metadata under the specified path
Online backups:
For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set
to a path where backup files are supposed to be saved, and the path specified in
this command must be within this base path`,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "offline",
Usage: "create backup without the node running",
},
},
ArgsUsage: "[backup file path]",
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 1 {
return xerrors.Errorf("expected 1 argument")
}
if cctx.Bool("offline") {
return offlineBackup(cctx)
}
return onlineBackup(cctx)
},
}
}
+6 -153
View File
@@ -8,7 +8,6 @@ import (
"os"
"os/exec"
"path"
"sort"
"strconv"
"strings"
"time"
@@ -28,10 +27,9 @@ import (
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/stmgr"
types "github.com/filecoin-project/lotus/chain/types"
)
@@ -52,7 +50,6 @@ var chainCmd = &cli.Command{
chainExportCmd,
slashConsensusFault,
chainGasPriceCmd,
chainInspectUsage,
},
}
@@ -162,7 +159,7 @@ var chainGetBlock = &cli.Command{
},
}
func apiMsgCids(in []lapi.Message) []cid.Cid {
func apiMsgCids(in []api.Message) []cid.Cid {
out := make([]cid.Cid, len(in))
for k, v := range in {
out[k] = v.Cid
@@ -378,143 +375,6 @@ var chainSetHeadCmd = &cli.Command{
},
}
var chainInspectUsage = &cli.Command{
Name: "inspect-usage",
Usage: "Inspect block space usage of a given tipset",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "tipset",
Usage: "specify tipset to view block space usage of",
Value: "@head",
},
&cli.IntFlag{
Name: "length",
Usage: "length of chain to inspect block space usage for",
Value: 1,
},
&cli.IntFlag{
Name: "num-results",
Usage: "number of results to print per category",
Value: 10,
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
ts, err := LoadTipSet(ctx, cctx, api)
if err != nil {
return err
}
cur := ts
var msgs []lapi.Message
for i := 0; i < cctx.Int("length"); i++ {
pmsgs, err := api.ChainGetParentMessages(ctx, cur.Blocks()[0].Cid())
if err != nil {
return err
}
msgs = append(msgs, pmsgs...)
next, err := api.ChainGetTipSet(ctx, cur.Parents())
if err != nil {
return err
}
cur = next
}
codeCache := make(map[address.Address]cid.Cid)
lookupActorCode := func(a address.Address) (cid.Cid, error) {
c, ok := codeCache[a]
if ok {
return c, nil
}
act, err := api.StateGetActor(ctx, a, ts.Key())
if err != nil {
return cid.Undef, err
}
codeCache[a] = act.Code
return act.Code, nil
}
bySender := make(map[string]int64)
byDest := make(map[string]int64)
byMethod := make(map[string]int64)
var sum int64
for _, m := range msgs {
bySender[m.Message.From.String()] += m.Message.GasLimit
byDest[m.Message.To.String()] += m.Message.GasLimit
sum += m.Message.GasLimit
code, err := lookupActorCode(m.Message.To)
if err != nil {
return err
}
mm := stmgr.MethodsMap[code][m.Message.Method]
byMethod[mm.Name] += m.Message.GasLimit
}
type keyGasPair struct {
Key string
Gas int64
}
mapToSortedKvs := func(m map[string]int64) []keyGasPair {
var vals []keyGasPair
for k, v := range m {
vals = append(vals, keyGasPair{
Key: k,
Gas: v,
})
}
sort.Slice(vals, func(i, j int) bool {
return vals[i].Gas > vals[j].Gas
})
return vals
}
senderVals := mapToSortedKvs(bySender)
destVals := mapToSortedKvs(byDest)
methodVals := mapToSortedKvs(byMethod)
numRes := cctx.Int("num-results")
fmt.Printf("Total Gas Limit: %d\n", sum)
fmt.Printf("By Sender:\n")
for i := 0; i < numRes && i < len(senderVals); i++ {
sv := senderVals[i]
fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas)
}
fmt.Println()
fmt.Printf("By Receiver:\n")
for i := 0; i < numRes && i < len(destVals); i++ {
sv := destVals[i]
fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas)
}
fmt.Println()
fmt.Printf("By Method:\n")
for i := 0; i < numRes && i < len(methodVals); i++ {
sv := methodVals[i]
fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas)
}
return nil
},
}
var chainListCmd = &cli.Command{
Name: "list",
Usage: "View a segment of the chain",
@@ -788,7 +648,7 @@ var chainGetCmd = &cli.Command{
type apiIpldStore struct {
ctx context.Context
api lapi.FullNode
api api.FullNode
}
func (ht *apiIpldStore) Context() context.Context {
@@ -816,7 +676,7 @@ func (ht *apiIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error)
panic("No mutations allowed")
}
func handleAmt(ctx context.Context, api lapi.FullNode, r cid.Cid) error {
func handleAmt(ctx context.Context, api api.FullNode, r cid.Cid) error {
s := &apiIpldStore{ctx, api}
mp, err := adt.AsArray(s, r)
if err != nil {
@@ -829,7 +689,7 @@ func handleAmt(ctx context.Context, api lapi.FullNode, r cid.Cid) error {
})
}
func handleHamtEpoch(ctx context.Context, api lapi.FullNode, r cid.Cid) error {
func handleHamtEpoch(ctx context.Context, api api.FullNode, r cid.Cid) error {
s := &apiIpldStore{ctx, api}
mp, err := adt.AsMap(s, r)
if err != nil {
@@ -847,7 +707,7 @@ func handleHamtEpoch(ctx context.Context, api lapi.FullNode, r cid.Cid) error {
})
}
func handleHamtAddress(ctx context.Context, api lapi.FullNode, r cid.Cid) error {
func handleHamtAddress(ctx context.Context, api api.FullNode, r cid.Cid) error {
s := &apiIpldStore{ctx, api}
mp, err := adt.AsMap(s, r)
if err != nil {
@@ -1070,20 +930,13 @@ var chainExportCmd = &cli.Command{
return err
}
var last bool
for b := range stream {
last = len(b) == 0
_, err := fi.Write(b)
if err != nil {
return err
}
}
if !last {
return xerrors.Errorf("incomplete export (remote connection lost?)")
}
return nil
},
}
+5 -58
View File
@@ -12,12 +12,13 @@ import (
"text/tabwriter"
"time"
"github.com/filecoin-project/specs-actors/actors/builtin"
tm "github.com/buger/goterm"
"github.com/docker/go-units"
"github.com/fatih/color"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-cidutil/cidenc"
"github.com/libp2p/go-libp2p-core/peer"
@@ -475,7 +476,6 @@ func interactiveDeal(cctx *cli.Context) error {
var ask storagemarket.StorageAsk
var epochPrice big.Int
var epochs abi.ChainEpoch
var verified bool
var a address.Address
if from := cctx.String("from"); from != "" {
@@ -572,53 +572,6 @@ func interactiveDeal(cctx *cli.Context) error {
ask = *a
// TODO: run more validation
state = "verified"
case "verified":
ts, err := api.ChainHead(ctx)
if err != nil {
return err
}
dcap, err := api.StateVerifiedClientStatus(ctx, a, ts.Key())
if err != nil {
return err
}
if dcap == nil {
state = "confirm"
continue
}
color.Blue(".. checking verified deal eligibility\n")
ds, err := api.ClientDealSize(ctx, data)
if err != nil {
return err
}
if dcap.Uint64() < uint64(ds.PieceSize) {
color.Yellow(".. not enough DataCap available for a verified deal\n")
state = "confirm"
continue
}
fmt.Print("\nMake this a verified deal? (yes/no): ")
var yn string
_, err = fmt.Scan(&yn)
if err != nil {
return err
}
switch yn {
case "yes":
verified = true
case "no":
verified = false
default:
fmt.Println("Type in full 'yes' or 'no'")
continue
}
state = "confirm"
case "confirm":
fromBal, err := api.WalletBalance(ctx, a)
@@ -637,15 +590,10 @@ func interactiveDeal(cctx *cli.Context) error {
epochs = abi.ChainEpoch(dur / (time.Duration(build.BlockDelaySecs) * time.Second))
// TODO: do some more or epochs math (round to miner PP, deal start buffer)
pricePerGib := ask.Price
if verified {
pricePerGib = ask.VerifiedPrice
}
gib := types.NewInt(1 << 30)
// TODO: price is based on PaddedPieceSize, right?
epochPrice = types.BigDiv(types.BigMul(pricePerGib, types.NewInt(uint64(ds.PieceSize))), gib)
epochPrice = types.BigDiv(types.BigMul(ask.Price, types.NewInt(uint64(ds.PieceSize))), gib)
totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs)))
fmt.Printf("-----\n")
@@ -655,7 +603,6 @@ func interactiveDeal(cctx *cli.Context) error {
fmt.Printf("Piece size: %s (Payload size: %s)\n", units.BytesSize(float64(ds.PieceSize)), units.BytesSize(float64(ds.PayloadSize)))
fmt.Printf("Duration: %s\n", dur)
fmt.Printf("Total price: ~%s (%s per epoch)\n", types.FIL(totalPrice), types.FIL(epochPrice))
fmt.Printf("Verified: %v\n", verified)
state = "accept"
case "accept":
@@ -690,7 +637,7 @@ func interactiveDeal(cctx *cli.Context) error {
MinBlocksDuration: uint64(epochs),
DealStartEpoch: abi.ChainEpoch(cctx.Int64("start-epoch")),
FastRetrieval: cctx.Bool("fast-retrieval"),
VerifiedDeal: verified,
VerifiedDeal: false, // TODO: Allow setting
})
if err != nil {
return err
@@ -1476,7 +1423,7 @@ func toChannelOutput(useColor bool, otherPartyColumn string, channel lapi.DataTr
otherPartyColumn: otherParty,
"Root Cid": rootCid,
"Initiated?": initiated,
"Transferred": units.BytesSize(float64(channel.Transferred)),
"Transferred": channel.Transferred,
"Voucher": voucher,
"Message": channel.Message,
}
-15
View File
@@ -216,13 +216,6 @@ func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
log.Errorf("repoType type does not match the type of repo.RepoType")
}
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
return tn.(api.StorageMiner), func() {}, nil
}
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
return tn.(api.FullNode), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, t)
if err != nil {
return nil, nil, err
@@ -232,10 +225,6 @@ func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) {
}
func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) {
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
return tn.(api.FullNode), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, repo.FullNode)
if err != nil {
return nil, nil, err
@@ -245,10 +234,6 @@ func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error
}
func GetStorageMinerAPI(ctx *cli.Context, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
return tn.(api.StorageMiner), func() {}, nil
}
addr, headers, err := GetRawAPI(ctx, repo.StorageMiner)
if err != nil {
return nil, nil, err
+40 -94
View File
@@ -7,7 +7,6 @@ import (
"sort"
"strconv"
cid "github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
@@ -44,10 +43,6 @@ var mpoolPending = &cli.Command{
Name: "local",
Usage: "print pending messages for addresses in local wallet only",
},
&cli.BoolFlag{
Name: "cids",
Usage: "only print cids of messages in output",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
@@ -84,15 +79,11 @@ var mpoolPending = &cli.Command{
}
}
if cctx.Bool("cids") {
fmt.Println(msg.Cid())
} else {
out, err := json.MarshalIndent(msg, "", " ")
if err != nil {
return err
}
fmt.Println(string(out))
out, err := json.MarshalIndent(msg, "", " ")
if err != nil {
return err
}
fmt.Println(string(out))
}
return nil
@@ -164,6 +155,14 @@ var mpoolSub = &cli.Command{
},
}
type statBucket struct {
msgs map[uint64]*types.SignedMessage
}
type mpStat struct {
addr string
past, cur, future uint64
}
var mpoolStat = &cli.Command{
Name: "stat",
Usage: "print mempool stats",
@@ -172,11 +171,6 @@ var mpoolStat = &cli.Command{
Name: "local",
Usage: "print stats for addresses in local wallet only",
},
&cli.IntFlag{
Name: "basefee-lookback",
Usage: "number of blocks to look back for minimum basefee",
Value: 60,
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
@@ -191,20 +185,6 @@ var mpoolStat = &cli.Command{
if err != nil {
return xerrors.Errorf("getting chain head: %w", err)
}
currBF := ts.Blocks()[0].ParentBaseFee
minBF := currBF
{
currTs := ts
for i := 0; i < cctx.Int("basefee-lookback"); i++ {
currTs, err = api.ChainGetTipSet(ctx, currTs.Parents())
if err != nil {
return xerrors.Errorf("walking chain: %w", err)
}
if newBF := currTs.Blocks()[0].ParentBaseFee; newBF.LessThan(minBF) {
minBF = newBF
}
}
}
var filter map[address.Address]struct{}
if cctx.Bool("local") {
@@ -225,16 +205,8 @@ var mpoolStat = &cli.Command{
return err
}
type statBucket struct {
msgs map[uint64]*types.SignedMessage
}
type mpStat struct {
addr string
past, cur, future uint64
belowCurr, belowPast uint64
}
buckets := map[address.Address]*statBucket{}
for _, v := range msgs {
if filter != nil {
if _, has := filter[v.Message.From]; !has {
@@ -271,27 +243,23 @@ var mpoolStat = &cli.Command{
cur++
}
var s mpStat
s.addr = a.String()
past := uint64(0)
future := uint64(0)
for _, m := range bkt.msgs {
if m.Message.Nonce < act.Nonce {
s.past++
} else if m.Message.Nonce > cur {
s.future++
} else {
s.cur++
past++
}
if m.Message.GasFeeCap.LessThan(currBF) {
s.belowCurr++
}
if m.Message.GasFeeCap.LessThan(minBF) {
s.belowPast++
if m.Message.Nonce > cur {
future++
}
}
out = append(out, s)
out = append(out, mpStat{
addr: a.String(),
past: past,
cur: cur - act.Nonce,
future: future,
})
}
sort.Slice(out, func(i, j int) bool {
@@ -304,14 +272,12 @@ var mpoolStat = &cli.Command{
total.past += stat.past
total.cur += stat.cur
total.future += stat.future
total.belowCurr += stat.belowCurr
total.belowPast += stat.belowPast
fmt.Printf("%s: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d \n", stat.addr, stat.past, stat.cur, stat.future, stat.belowCurr, cctx.Int("basefee-lookback"), stat.belowPast)
fmt.Printf("%s: past: %d, cur: %d, future: %d\n", stat.addr, stat.past, stat.cur, stat.future)
}
fmt.Println("-----")
fmt.Printf("total: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d \n", total.past, total.cur, total.future, total.belowCurr, cctx.Int("basefee-lookback"), total.belowPast)
fmt.Printf("total: past: %d, cur: %d, future: %d\n", total.past, total.cur, total.future)
return nil
},
@@ -342,8 +308,21 @@ var mpoolReplaceCmd = &cli.Command{
Usage: "Spend up to X FIL for this message (applicable for auto mode)",
},
},
ArgsUsage: "<from nonce> | <message-cid>",
ArgsUsage: "[from] [nonce]",
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() < 2 {
return cli.ShowCommandHelp(cctx, cctx.Command.Name)
}
from, err := address.NewFromString(cctx.Args().Get(0))
if err != nil {
return err
}
nonce, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64)
if err != nil {
return err
}
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
@@ -353,39 +332,6 @@ var mpoolReplaceCmd = &cli.Command{
ctx := ReqContext(cctx)
var from address.Address
var nonce uint64
switch cctx.Args().Len() {
case 1:
mcid, err := cid.Decode(cctx.Args().First())
if err != nil {
return err
}
msg, err := api.ChainGetMessage(ctx, mcid)
if err != nil {
return fmt.Errorf("could not find referenced message: %w", err)
}
from = msg.From
nonce = msg.Nonce
case 2:
f, err := address.NewFromString(cctx.Args().Get(0))
if err != nil {
return err
}
n, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64)
if err != nil {
return err
}
from = f
nonce = n
default:
return cli.ShowCommandHelp(cctx, cctx.Command.Name)
}
ts, err := api.ChainHead(ctx)
if err != nil {
return xerrors.Errorf("getting chain head: %w", err)
+2 -3
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"time"
"github.com/hako/durafmt"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
@@ -37,11 +36,11 @@ func parseTipSet(ctx context.Context, api api.FullNode, vals []string) (*types.T
func EpochTime(curr, e abi.ChainEpoch) string {
switch {
case curr > e:
return fmt.Sprintf("%d (%s ago)", e, durafmt.Parse(time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e))).LimitFirstN(2))
return fmt.Sprintf("%d (%s ago)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e)))
case curr == e:
return fmt.Sprintf("%d (now)", e)
case curr < e:
return fmt.Sprintf("%d (in %s)", e, durafmt.Parse(time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr))).LimitFirstN(2))
return fmt.Sprintf("%d (in %s)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr)))
}
panic("math broke")
+1 -1
View File
@@ -86,5 +86,5 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci
// TODO: additional anti-spam checks
return a.api.MpoolPushUntrusted(ctx, sm)
return a.api.MpoolPush(ctx, sm)
}
+51 -599
View File
@@ -1,10 +1,8 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/csv"
"fmt"
"io/ioutil"
"net/http"
@@ -29,12 +27,11 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
@@ -47,8 +44,6 @@ var log = logging.Logger("main")
func main() {
local := []*cli.Command{
runCmd,
recoverMinersCmd,
findMinersCmd,
versionCmd,
}
@@ -112,186 +107,6 @@ var versionCmd = &cli.Command{
},
}
var findMinersCmd = &cli.Command{
Name: "find-miners",
Usage: "find miners with a desired minimum balance",
Description: `Find miners returns a list of miners and their balances that are below a
threhold value. By default only the miner actor available balance is considered but other
account balances can be included by enabling them through the flags.
Examples
Find all miners with an available balance below 100 FIL
lotus-pcr find-miners --threshold 100
Find all miners with a balance below zero, which includes the owner and worker balances
lotus-pcr find-miners --threshold 0 --owner --worker
`,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-sync",
EnvVars: []string{"LOTUS_PCR_NO_SYNC"},
Usage: "do not wait for chain sync to complete",
},
&cli.IntFlag{
Name: "threshold",
EnvVars: []string{"LOTUS_PCR_THRESHOLD"},
Usage: "balance below this limit will be printed",
Value: 0,
},
&cli.BoolFlag{
Name: "owner",
Usage: "include owner balance",
Value: false,
},
&cli.BoolFlag{
Name: "worker",
Usage: "include worker balance",
Value: false,
},
&cli.BoolFlag{
Name: "control",
Usage: "include control balance",
Value: false,
},
},
Action: func(cctx *cli.Context) error {
ctx := context.Background()
api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path"))
if err != nil {
log.Fatal(err)
}
defer closer()
if !cctx.Bool("no-sync") {
if err := stats.WaitForSyncComplete(ctx, api); err != nil {
log.Fatal(err)
}
}
owner := cctx.Bool("owner")
worker := cctx.Bool("worker")
control := cctx.Bool("control")
threshold := uint64(cctx.Int("threshold"))
rf := &refunder{
api: api,
threshold: types.FromFil(threshold),
}
refundTipset, err := api.ChainHead(ctx)
if err != nil {
return err
}
balanceRefund, err := rf.FindMiners(ctx, refundTipset, NewMinersRefund(), owner, worker, control)
if err != nil {
return err
}
for _, maddr := range balanceRefund.Miners() {
fmt.Printf("%s\t%s\n", maddr, types.FIL(balanceRefund.GetRefund(maddr)))
}
return nil
},
}
var recoverMinersCmd = &cli.Command{
Name: "recover-miners",
Usage: "Ensure all miners with a negative available balance have a FIL surplus across accounts",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "from",
EnvVars: []string{"LOTUS_PCR_FROM"},
Usage: "wallet address to send refund from",
},
&cli.BoolFlag{
Name: "no-sync",
EnvVars: []string{"LOTUS_PCR_NO_SYNC"},
Usage: "do not wait for chain sync to complete",
},
&cli.BoolFlag{
Name: "dry-run",
EnvVars: []string{"LOTUS_PCR_DRY_RUN"},
Usage: "do not send any messages",
Value: false,
},
&cli.StringFlag{
Name: "output",
Usage: "dump data as a csv format to this file",
},
&cli.IntFlag{
Name: "miner-recovery-cutoff",
EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_CUTOFF"},
Usage: "maximum amount of FIL that can be sent to any one miner before refund percent is applied",
Value: 3000,
},
&cli.IntFlag{
Name: "miner-recovery-bonus",
EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_BONUS"},
Usage: "additional FIL to send to each miner",
Value: 5,
},
&cli.IntFlag{
Name: "miner-recovery-refund-percent",
EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_REFUND_PERCENT"},
Usage: "percent of refund to issue",
Value: 110,
},
},
Action: func(cctx *cli.Context) error {
ctx := context.Background()
api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path"))
if err != nil {
log.Fatal(err)
}
defer closer()
from, err := address.NewFromString(cctx.String("from"))
if err != nil {
return xerrors.Errorf("parsing source address (provide correct --from flag!): %w", err)
}
if !cctx.Bool("no-sync") {
if err := stats.WaitForSyncComplete(ctx, api); err != nil {
log.Fatal(err)
}
}
dryRun := cctx.Bool("dry-run")
minerRecoveryRefundPercent := cctx.Int("miner-recovery-refund-percent")
minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff"))
minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus"))
rf := &refunder{
api: api,
wallet: from,
dryRun: dryRun,
minerRecoveryRefundPercent: minerRecoveryRefundPercent,
minerRecoveryCutoff: types.FromFil(minerRecoveryCutoff),
minerRecoveryBonus: types.FromFil(minerRecoveryBonus),
}
refundTipset, err := api.ChainHead(ctx)
if err != nil {
return err
}
balanceRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund(), cctx.String("output"))
if err != nil {
return err
}
if err := rf.Refund(ctx, "refund to recover miner", refundTipset, balanceRefund, 0); err != nil {
return err
}
return nil
},
}
var runCmd = &cli.Command{
Name: "run",
Usage: "Start message reimpursement",
@@ -307,10 +122,10 @@ var runCmd = &cli.Command{
Usage: "do not wait for chain sync to complete",
},
&cli.IntFlag{
Name: "refund-percent",
EnvVars: []string{"LOTUS_PCR_REFUND_PERCENT"},
Usage: "percent of refund to issue",
Value: 103,
Name: "percent-extra",
EnvVars: []string{"LOTUS_PCR_PERCENT_EXTRA"},
Usage: "extra funds to send above the refund",
Value: 3,
},
&cli.IntFlag{
Name: "max-message-queue",
@@ -348,36 +163,6 @@ var runCmd = &cli.Command{
Usage: "the number of tipsets to delay message processing to smooth chain reorgs",
Value: int(build.MessageConfidence),
},
&cli.BoolFlag{
Name: "miner-recovery",
EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY"},
Usage: "run the miner recovery job",
Value: false,
},
&cli.IntFlag{
Name: "miner-recovery-period",
EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_PERIOD"},
Usage: "interval between running miner recovery",
Value: 2880,
},
&cli.IntFlag{
Name: "miner-recovery-cutoff",
EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_CUTOFF"},
Usage: "maximum amount of FIL that can be sent to any one miner before refund percent is applied",
Value: 3000,
},
&cli.IntFlag{
Name: "miner-recovery-bonus",
EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_BONUS"},
Usage: "additional FIL to send to each miner",
Value: 5,
},
&cli.IntFlag{
Name: "miner-recovery-refund-percent",
EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_REFUND_PERCENT"},
Usage: "percent of refund to issue",
Value: 110,
},
},
Action: func(cctx *cli.Context) error {
go func() {
@@ -416,33 +201,24 @@ var runCmd = &cli.Command{
log.Fatal(err)
}
refundPercent := cctx.Int("refund-percent")
percentExtra := cctx.Int("percent-extra")
maxMessageQueue := cctx.Int("max-message-queue")
dryRun := cctx.Bool("dry-run")
preCommitEnabled := cctx.Bool("pre-commit")
proveCommitEnabled := cctx.Bool("prove-commit")
aggregateTipsets := cctx.Int("aggregate-tipsets")
minerRecoveryEnabled := cctx.Bool("miner-recovery")
minerRecoveryPeriod := abi.ChainEpoch(int64(cctx.Int("miner-recovery-period")))
minerRecoveryRefundPercent := cctx.Int("miner-recovery-refund-percent")
minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff"))
minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus"))
rf := &refunder{
api: api,
wallet: from,
refundPercent: refundPercent,
minerRecoveryRefundPercent: minerRecoveryRefundPercent,
minerRecoveryCutoff: types.FromFil(minerRecoveryCutoff),
minerRecoveryBonus: types.FromFil(minerRecoveryBonus),
dryRun: dryRun,
preCommitEnabled: preCommitEnabled,
proveCommitEnabled: proveCommitEnabled,
api: api,
wallet: from,
percentExtra: percentExtra,
dryRun: dryRun,
preCommitEnabled: preCommitEnabled,
proveCommitEnabled: proveCommitEnabled,
}
var refunds *MinersRefund = NewMinersRefund()
var rounds int = 0
nextMinerRecovery := r.MinerRecoveryHeight() + minerRecoveryPeriod
for tipset := range tipsetsCh {
refunds, err = rf.ProcessTipset(ctx, tipset, refunds)
@@ -450,34 +226,17 @@ var runCmd = &cli.Command{
return err
}
refundTipset, err := api.ChainHead(ctx)
if err != nil {
return err
}
if minerRecoveryEnabled && refundTipset.Height() >= nextMinerRecovery {
recoveryRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund(), "")
if err != nil {
return err
}
if err := rf.Refund(ctx, "refund to recover miners", refundTipset, recoveryRefund, 0); err != nil {
return err
}
if err := r.SetMinerRecoveryHeight(tipset.Height()); err != nil {
return err
}
nextMinerRecovery = r.MinerRecoveryHeight() + minerRecoveryPeriod
}
rounds = rounds + 1
if rounds < aggregateTipsets {
continue
}
if err := rf.Refund(ctx, "refund stats", refundTipset, refunds, rounds); err != nil {
refundTipset, err := api.ChainHead(ctx)
if err != nil {
return err
}
if err := rf.Refund(ctx, refundTipset, refunds, rounds); err != nil {
return err
}
@@ -536,6 +295,7 @@ func (m *MinersRefund) Track(addr address.Address, value types.BigInt) {
m.count = m.count + 1
m.totalRefunds = types.BigAdd(m.totalRefunds, value)
m.refunds[addr] = types.BigAdd(m.refunds[addr], value)
}
@@ -564,14 +324,8 @@ type refunderNodeApi interface {
ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]api.Message, error)
ChainGetParentReceipts(ctx context.Context, blockCid cid.Cid) ([]*types.MessageReceipt, error)
ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
StateMinerInitialPledgeCollateral(ctx context.Context, addr address.Address, precommitInfo miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error)
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error)
StateSectorPreCommitInfo(ctx context.Context, addr address.Address, sector abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, tsk types.TipSetKey) ([]*miner.SectorOnChainInfo, error)
StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) (bitfield.BitField, error)
StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error)
StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error)
StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error)
MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error)
@@ -580,241 +334,12 @@ type refunderNodeApi interface {
}
type refunder struct {
api refunderNodeApi
wallet address.Address
refundPercent int
minerRecoveryRefundPercent int
minerRecoveryCutoff big.Int
minerRecoveryBonus big.Int
dryRun bool
preCommitEnabled bool
proveCommitEnabled bool
threshold big.Int
}
func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) {
miners, err := r.api.StateListMiners(ctx, tipset.Key())
if err != nil {
return nil, err
}
for _, maddr := range miners {
mact, err := r.api.StateGetActor(ctx, maddr, types.EmptyTSK)
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
if !mact.Balance.GreaterThan(big.Zero()) {
continue
}
minerAvailableBalance, err := r.api.StateMinerAvailableBalance(ctx, maddr, tipset.Key())
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
// Look up and find all addresses associated with the miner
minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key())
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
allAddresses := []address.Address{}
if worker {
allAddresses = append(allAddresses, minerInfo.Worker)
}
if owner {
allAddresses = append(allAddresses, minerInfo.Owner)
}
if control {
allAddresses = append(allAddresses, minerInfo.ControlAddresses...)
}
// Sum the balancer of all the addresses
addrSum := big.Zero()
addrCheck := make(map[address.Address]struct{}, len(allAddresses))
for _, addr := range allAddresses {
if _, found := addrCheck[addr]; !found {
balance, err := r.api.WalletBalance(ctx, addr)
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
addrSum = big.Add(addrSum, balance)
addrCheck[addr] = struct{}{}
}
}
totalAvailableBalance := big.Add(addrSum, minerAvailableBalance)
if totalAvailableBalance.GreaterThanEqual(r.threshold) {
continue
}
refunds.Track(maddr, totalAvailableBalance)
log.Debugw("processing miner", "miner", maddr, "sectors", "available_balance", totalAvailableBalance)
}
return refunds, nil
}
func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, output string) (*MinersRefund, error) {
miners, err := r.api.StateListMiners(ctx, tipset.Key())
if err != nil {
return nil, err
}
w := ioutil.Discard
if len(output) != 0 {
f, err := os.Create(output)
if err != nil {
return nil, err
}
defer f.Close() // nolint:errcheck
w = bufio.NewWriter(f)
}
csvOut := csv.NewWriter(w)
defer csvOut.Flush()
if err := csvOut.Write([]string{"MinerID", "FaultedSectors", "AvailableBalance", "ProposedRefund"}); err != nil {
return nil, err
}
for _, maddr := range miners {
mact, err := r.api.StateGetActor(ctx, maddr, types.EmptyTSK)
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
if !mact.Balance.GreaterThan(big.Zero()) {
continue
}
minerAvailableBalance, err := r.api.StateMinerAvailableBalance(ctx, maddr, tipset.Key())
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
// Look up and find all addresses associated with the miner
minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key())
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
allAddresses := []address.Address{minerInfo.Worker, minerInfo.Owner}
allAddresses = append(allAddresses, minerInfo.ControlAddresses...)
// Sum the balancer of all the addresses
addrSum := big.Zero()
addrCheck := make(map[address.Address]struct{}, len(allAddresses))
for _, addr := range allAddresses {
if _, found := addrCheck[addr]; !found {
balance, err := r.api.WalletBalance(ctx, addr)
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
addrSum = big.Add(addrSum, balance)
addrCheck[addr] = struct{}{}
}
}
faults, err := r.api.StateMinerFaults(ctx, maddr, tipset.Key())
if err != nil {
log.Errorw("failed to look up miner faults", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
faultsCount, err := faults.Count()
if err != nil {
log.Errorw("failed to get count of faults", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
if faultsCount == 0 {
log.Debugw("skipping miner with zero faults", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}
totalAvailableBalance := big.Add(addrSum, minerAvailableBalance)
balanceCutoff := big.Mul(big.Div(big.NewIntUnsigned(faultsCount), big.NewInt(10)), big.NewIntUnsigned(build.FilecoinPrecision))
if totalAvailableBalance.GreaterThan(balanceCutoff) {
log.Debugw(
"skipping over miner with total available balance larger than refund",
"height", tipset.Height(),
"key", tipset.Key(),
"miner", maddr,
"available_balance", totalAvailableBalance,
"balance_cutoff", balanceCutoff,
"faults_count", faultsCount,
"available_balance_fil", big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
"balance_cutoff_fil", big.Div(balanceCutoff, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
)
continue
}
refundValue := big.Sub(balanceCutoff, totalAvailableBalance)
if r.minerRecoveryRefundPercent > 0 {
refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.minerRecoveryRefundPercent)))
}
refundValue = big.Add(refundValue, r.minerRecoveryBonus)
if refundValue.GreaterThan(r.minerRecoveryCutoff) {
log.Infow(
"skipping over miner with refund greater than refund cutoff",
"height", tipset.Height(),
"key", tipset.Key(),
"miner", maddr,
"available_balance", totalAvailableBalance,
"balance_cutoff", balanceCutoff,
"faults_count", faultsCount,
"refund", refundValue,
"available_balance_fil", big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
"balance_cutoff_fil", big.Div(balanceCutoff, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
"refund_fil", big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
)
continue
}
refunds.Track(maddr, refundValue)
record := []string{
maddr.String(),
fmt.Sprintf("%d", faultsCount),
big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).String(),
big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).String(),
}
if err := csvOut.Write(record); err != nil {
return nil, err
}
log.Debugw(
"processing miner",
"miner", maddr,
"faults_count", faultsCount,
"available_balance", totalAvailableBalance,
"refund", refundValue,
"available_balance_fil", big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
"refund_fil", big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
)
}
return refunds, nil
api refunderNodeApi
wallet address.Address
percentExtra int
dryRun bool
preCommitEnabled bool
proveCommitEnabled bool
}
func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) {
@@ -935,41 +460,22 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu
continue
}
if r.refundPercent > 0 {
refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent)))
if r.percentExtra > 0 {
refundValue = types.BigAdd(refundValue, types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.percentExtra))))
}
log.Debugw(
"processing message",
"method", messageMethod,
"cid", msg.Cid,
"from", m.From,
"to", m.To,
"value", m.Value,
"gas_fee_cap", m.GasFeeCap,
"gas_premium", m.GasPremium,
"gas_used", recps[i].GasUsed,
"refund", refundValue,
"refund_fil", big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
)
log.Debugw("processing message", "method", messageMethod, "cid", msg.Cid, "from", m.From, "to", m.To, "value", m.Value, "gas_fee_cap", m.GasFeeCap, "gas_premium", m.GasPremium, "gas_used", recps[i].GasUsed, "refund", refundValue)
refunds.Track(m.From, refundValue)
tipsetRefunds.Track(m.From, refundValue)
}
log.Infow(
"tipset stats",
"height", tipset.Height(),
"key", tipset.Key(),
"total_refunds", tipsetRefunds.TotalRefunds(),
"total_refunds_fil", big.Div(tipsetRefunds.TotalRefunds(), big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
"messages_processed", tipsetRefunds.Count(),
)
log.Infow("tipset stats", "height", tipset.Height(), "key", tipset.Key(), "total_refunds", tipsetRefunds.TotalRefunds(), "messages_processed", tipsetRefunds.Count())
return refunds, nil
}
func (r *refunder) Refund(ctx context.Context, name string, tipset *types.TipSet, refunds *MinersRefund, rounds int) error {
func (r *refunder) Refund(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, rounds int) error {
if refunds.Count() == 0 {
log.Debugw("no messages to refund in tipset", "height", tipset.Height(), "key", tipset.Key())
return nil
@@ -1027,24 +533,13 @@ func (r *refunder) Refund(ctx context.Context, name string, tipset *types.TipSet
refundSum = types.BigAdd(refundSum, msg.Value)
}
log.Infow(
name,
"tipsets_processed", rounds,
"height", tipset.Height(),
"key", tipset.Key(),
"messages_sent", len(messages)-failures,
"refund_sum", refundSum,
"refund_sum_fil", big.Div(refundSum, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(),
"messages_failures", failures,
"messages_processed", refunds.Count(),
)
log.Infow("refund stats", "tipsets_processed", rounds, "height", tipset.Height(), "key", tipset.Key(), "messages_sent", len(messages)-failures, "refund_sum", refundSum, "messages_failures", failures, "messages_processed", refunds.Count())
return nil
}
type Repo struct {
lastHeight abi.ChainEpoch
lastMinerRecoveryHeight abi.ChainEpoch
path string
last abi.ChainEpoch
path string
}
func NewRepo(path string) (*Repo, error) {
@@ -1054,9 +549,8 @@ func NewRepo(path string) (*Repo, error) {
}
return &Repo{
lastHeight: 0,
lastMinerRecoveryHeight: 0,
path: path,
last: 0,
path: path,
}, nil
}
@@ -1087,66 +581,43 @@ func (r *Repo) init() error {
return nil
}
func (r *Repo) Open() error {
if err := r.init(); err != nil {
return err
func (r *Repo) Open() (err error) {
if err = r.init(); err != nil {
return
}
if err := r.loadHeight(); err != nil {
return err
}
var f *os.File
if err := r.loadMinerRecoveryHeight(); err != nil {
return err
}
return nil
}
func loadChainEpoch(fn string) (abi.ChainEpoch, error) {
f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE, 0644)
f, err = os.OpenFile(filepath.Join(r.path, "height"), os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return 0, err
return
}
defer func() {
err = f.Close()
}()
raw, err := ioutil.ReadAll(f)
var raw []byte
raw, err = ioutil.ReadAll(f)
if err != nil {
return 0, err
return
}
height, err := strconv.Atoi(string(bytes.TrimSpace(raw)))
if err != nil {
return 0, err
return
}
return abi.ChainEpoch(height), nil
}
func (r *Repo) loadHeight() error {
var err error
r.lastHeight, err = loadChainEpoch(filepath.Join(r.path, "height"))
return err
}
func (r *Repo) loadMinerRecoveryHeight() error {
var err error
r.lastMinerRecoveryHeight, err = loadChainEpoch(filepath.Join(r.path, "miner_recovery_height"))
return err
r.last = abi.ChainEpoch(height)
return
}
func (r *Repo) Height() abi.ChainEpoch {
return r.lastHeight
}
func (r *Repo) MinerRecoveryHeight() abi.ChainEpoch {
return r.lastMinerRecoveryHeight
return r.last
}
func (r *Repo) SetHeight(last abi.ChainEpoch) (err error) {
r.lastHeight = last
r.last = last
var f *os.File
f, err = os.OpenFile(filepath.Join(r.path, "height"), os.O_RDWR, 0644)
if err != nil {
@@ -1157,26 +628,7 @@ func (r *Repo) SetHeight(last abi.ChainEpoch) (err error) {
err = f.Close()
}()
if _, err = fmt.Fprintf(f, "%d", r.lastHeight); err != nil {
return
}
return
}
func (r *Repo) SetMinerRecoveryHeight(last abi.ChainEpoch) (err error) {
r.lastMinerRecoveryHeight = last
var f *os.File
f, err = os.OpenFile(filepath.Join(r.path, "miner_recovery_height"), os.O_RDWR, 0644)
if err != nil {
return
}
defer func() {
err = f.Close()
}()
if _, err = fmt.Fprintf(f, "%d", r.lastMinerRecoveryHeight); err != nil {
if _, err = fmt.Fprintf(f, "%d", r.last); err != nil {
return
}
-290
View File
@@ -1,290 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strings"
"github.com/docker/go-units"
"github.com/ipfs/go-datastore"
dsq "github.com/ipfs/go-datastore/query"
logging "github.com/ipfs/go-log"
"github.com/polydawn/refmt/cbor"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/backupds"
"github.com/filecoin-project/lotus/node/repo"
)
var datastoreCmd = &cli.Command{
Name: "datastore",
Description: "access node datastores directly",
Subcommands: []*cli.Command{
datastoreBackupCmd,
datastoreListCmd,
datastoreGetCmd,
},
}
var datastoreListCmd = &cli.Command{
Name: "list",
Description: "list datastore keys",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "repo-type",
Usage: "node type (1 - full, 2 - storage, 3 - worker)",
Value: 1,
},
&cli.BoolFlag{
Name: "top-level",
Usage: "only print top-level keys",
},
&cli.StringFlag{
Name: "get-enc",
Usage: "print values [esc/hex/cbor]",
},
},
ArgsUsage: "[namespace prefix]",
Action: func(cctx *cli.Context) error {
logging.SetLogLevel("badger", "ERROR") // nolint:errcheck
r, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return xerrors.Errorf("opening fs repo: %w", err)
}
exists, err := r.Exists()
if err != nil {
return err
}
if !exists {
return xerrors.Errorf("lotus repo doesn't exist")
}
lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type")))
if err != nil {
return err
}
defer lr.Close() //nolint:errcheck
ds, err := lr.Datastore(datastore.NewKey(cctx.Args().First()).String())
if err != nil {
return err
}
genc := cctx.String("get-enc")
q, err := ds.Query(dsq.Query{
Prefix: datastore.NewKey(cctx.Args().Get(1)).String(),
KeysOnly: genc == "",
})
if err != nil {
return xerrors.Errorf("datastore query: %w", err)
}
defer q.Close() //nolint:errcheck
printKv := kvPrinter(cctx.Bool("top-level"), genc)
for res := range q.Next() {
if err := printKv(res.Key, res.Value); err != nil {
return err
}
}
return nil
},
}
var datastoreGetCmd = &cli.Command{
Name: "get",
Description: "list datastore keys",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "repo-type",
Usage: "node type (1 - full, 2 - storage, 3 - worker)",
Value: 1,
},
&cli.StringFlag{
Name: "enc",
Usage: "encoding (esc/hex/cbor)",
Value: "esc",
},
},
ArgsUsage: "[namespace key]",
Action: func(cctx *cli.Context) error {
logging.SetLogLevel("badger", "ERROR") // nolint:errchec
r, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return xerrors.Errorf("opening fs repo: %w", err)
}
exists, err := r.Exists()
if err != nil {
return err
}
if !exists {
return xerrors.Errorf("lotus repo doesn't exist")
}
lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type")))
if err != nil {
return err
}
defer lr.Close() //nolint:errcheck
ds, err := lr.Datastore(datastore.NewKey(cctx.Args().First()).String())
if err != nil {
return err
}
val, err := ds.Get(datastore.NewKey(cctx.Args().Get(1)))
if err != nil {
return xerrors.Errorf("get: %w", err)
}
return printVal(cctx.String("enc"), val)
},
}
var datastoreBackupCmd = &cli.Command{
Name: "backup",
Description: "manage datastore backups",
Subcommands: []*cli.Command{
datastoreBackupStatCmd,
datastoreBackupListCmd,
},
}
var datastoreBackupStatCmd = &cli.Command{
Name: "stat",
Description: "validate and print info about datastore backup",
ArgsUsage: "[file]",
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 1 {
return xerrors.Errorf("expected 1 argument")
}
f, err := os.Open(cctx.Args().First())
if err != nil {
return xerrors.Errorf("opening backup file: %w", err)
}
defer f.Close() // nolint:errcheck
var keys, kbytes, vbytes uint64
err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error {
keys++
kbytes += uint64(len(key.String()))
vbytes += uint64(len(value))
return nil
})
if err != nil {
return err
}
fmt.Println("Keys: ", keys)
fmt.Println("Key bytes: ", units.BytesSize(float64(kbytes)))
fmt.Println("Value bytes: ", units.BytesSize(float64(vbytes)))
return err
},
}
var datastoreBackupListCmd = &cli.Command{
Name: "list",
Description: "list data in a backup",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "top-level",
Usage: "only print top-level keys",
},
&cli.StringFlag{
Name: "get-enc",
Usage: "print values [esc/hex/cbor]",
},
},
ArgsUsage: "[file]",
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 1 {
return xerrors.Errorf("expected 1 argument")
}
f, err := os.Open(cctx.Args().First())
if err != nil {
return xerrors.Errorf("opening backup file: %w", err)
}
defer f.Close() // nolint:errcheck
printKv := kvPrinter(cctx.Bool("top-level"), cctx.String("get-enc"))
err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error {
return printKv(key.String(), value)
})
if err != nil {
return err
}
return err
},
}
func kvPrinter(toplevel bool, genc string) func(sk string, value []byte) error {
seen := map[string]struct{}{}
return func(s string, value []byte) error {
if toplevel {
k := datastore.NewKey(datastore.NewKey(s).List()[0])
if k.Type() != "" {
s = k.Type()
} else {
s = k.String()
}
_, has := seen[s]
if has {
return nil
}
seen[s] = struct{}{}
}
s = fmt.Sprintf("%q", s)
s = strings.Trim(s, "\"")
fmt.Println(s)
if genc != "" {
fmt.Print("\t")
if err := printVal(genc, value); err != nil {
return err
}
}
return nil
}
}
func printVal(enc string, val []byte) error {
switch enc {
case "esc":
s := fmt.Sprintf("%q", string(val))
s = strings.Trim(s, "\"")
fmt.Println(s)
case "hex":
fmt.Printf("%x\n", val)
case "cbor":
var out interface{}
if err := cbor.Unmarshal(cbor.DecodeOptions{}, val, &out); err != nil {
return xerrors.Errorf("unmarshaling cbor: %w", err)
}
s, err := json.Marshal(&out)
if err != nil {
return xerrors.Errorf("remarshaling as json: %w", err)
}
fmt.Println(string(s))
default:
return xerrors.New("unknown encoding")
}
return nil
}
-2
View File
@@ -39,8 +39,6 @@ func main() {
consensusCmd,
serveDealStatsCmd,
syncCmd,
stateTreePruneCmd,
datastoreCmd,
}
app := &cli.App{
+5 -1
View File
@@ -91,8 +91,12 @@ var noncefix = &cli.Command{
Value: types.NewInt(1),
Nonce: i,
}
smsg, err := api.WalletSignMessage(ctx, addr, msg)
if err != nil {
return err
}
_, err = api.MpoolPushMessage(ctx, msg, nil)
_, err = api.MpoolPush(ctx, smsg)
if err != nil {
return err
}
-289
View File
@@ -1,289 +0,0 @@
package main
import (
"context"
"fmt"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/lib/blockstore"
"github.com/filecoin-project/lotus/node/repo"
"github.com/ipfs/bbloom"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
)
type cidSet interface {
Add(cid.Cid)
Has(cid.Cid) bool
HasRaw([]byte) bool
Len() int
}
type bloomSet struct {
bloom *bbloom.Bloom
}
func newBloomSet(size int64) (*bloomSet, error) {
b, err := bbloom.New(float64(size), 3)
if err != nil {
return nil, err
}
return &bloomSet{bloom: b}, nil
}
func (bs *bloomSet) Add(c cid.Cid) {
bs.bloom.Add(c.Hash())
}
func (bs *bloomSet) Has(c cid.Cid) bool {
return bs.bloom.Has(c.Hash())
}
func (bs *bloomSet) HasRaw(b []byte) bool {
return bs.bloom.Has(b)
}
func (bs *bloomSet) Len() int {
return int(bs.bloom.ElementsAdded())
}
type mapSet struct {
m map[string]struct{}
}
func newMapSet() *mapSet {
return &mapSet{m: make(map[string]struct{})}
}
func (bs *mapSet) Add(c cid.Cid) {
bs.m[string(c.Hash())] = struct{}{}
}
func (bs *mapSet) Has(c cid.Cid) bool {
_, ok := bs.m[string(c.Hash())]
return ok
}
func (bs *mapSet) HasRaw(b []byte) bool {
_, ok := bs.m[string(b)]
return ok
}
func (bs *mapSet) Len() int {
return len(bs.m)
}
var stateTreePruneCmd = &cli.Command{
Name: "state-prune",
Description: "Deletes old state root data from local chainstore",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "repo",
Value: "~/.lotus",
},
&cli.Int64Flag{
Name: "keep-from-lookback",
Usage: "keep stateroots at or newer than the current height minus this lookback",
Value: 1800, // 2 x finality
},
&cli.IntFlag{
Name: "delete-up-to",
Usage: "delete up to the given number of objects (used to run a faster 'partial' sync)",
},
&cli.BoolFlag{
Name: "use-bloom-set",
Usage: "use a bloom filter for the 'good' set instead of a map, reduces memory usage but may not clean up as much",
},
&cli.BoolFlag{
Name: "dry-run",
Usage: "only enumerate the good set, don't do any deletions",
},
&cli.BoolFlag{
Name: "only-ds-gc",
Usage: "Only run datastore GC",
},
&cli.IntFlag{
Name: "gc-count",
Usage: "number of times to run gc",
Value: 20,
},
},
Action: func(cctx *cli.Context) error {
ctx := context.TODO()
fsrepo, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return err
}
lkrepo, err := fsrepo.Lock(repo.FullNode)
if err != nil {
return err
}
defer lkrepo.Close() //nolint:errcheck
ds, err := lkrepo.Datastore("/chain")
if err != nil {
return err
}
defer ds.Close() //nolint:errcheck
mds, err := lkrepo.Datastore("/metadata")
if err != nil {
return err
}
defer mds.Close() //nolint:errcheck
if cctx.Bool("only-ds-gc") {
gcds, ok := ds.(datastore.GCDatastore)
if ok {
fmt.Println("running datastore gc....")
for i := 0; i < cctx.Int("gc-count"); i++ {
if err := gcds.CollectGarbage(); err != nil {
return xerrors.Errorf("datastore GC failed: %w", err)
}
}
fmt.Println("gc complete!")
return nil
}
return fmt.Errorf("datastore doesnt support gc")
}
bs := blockstore.NewBlockstore(ds)
cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier))
if err := cs.Load(); err != nil {
return fmt.Errorf("loading chainstore: %w", err)
}
var goodSet cidSet
if cctx.Bool("use-bloom-set") {
bset, err := newBloomSet(10000000)
if err != nil {
return err
}
goodSet = bset
} else {
goodSet = newMapSet()
}
ts := cs.GetHeaviestTipSet()
rrLb := abi.ChainEpoch(cctx.Int64("keep-from-lookback"))
if err := cs.WalkSnapshot(ctx, ts, rrLb, true, func(c cid.Cid) error {
if goodSet.Len()%20 == 0 {
fmt.Printf("\renumerating keep set: %d ", goodSet.Len())
}
goodSet.Add(c)
return nil
}); err != nil {
return fmt.Errorf("snapshot walk failed: %w", err)
}
fmt.Println()
fmt.Printf("Successfully marked keep set! (%d objects)\n", goodSet.Len())
if cctx.Bool("dry-run") {
return nil
}
var b datastore.Batch
var batchCount int
markForRemoval := func(c cid.Cid) error {
if b == nil {
nb, err := ds.Batch()
if err != nil {
return fmt.Errorf("opening batch: %w", err)
}
b = nb
}
batchCount++
if err := b.Delete(dshelp.MultihashToDsKey(c.Hash())); err != nil {
return err
}
if batchCount > 100 {
if err := b.Commit(); err != nil {
return xerrors.Errorf("failed to commit batch deletes: %w", err)
}
b = nil
batchCount = 0
}
return nil
}
res, err := ds.Query(query.Query{KeysOnly: true})
if err != nil {
return xerrors.Errorf("failed to query datastore: %w", err)
}
dupTo := cctx.Int("delete-up-to")
var deleteCount int
var goodHits int
for {
v, ok := res.NextSync()
if !ok {
break
}
bk, err := dshelp.BinaryFromDsKey(datastore.RawKey(v.Key[len("/blocks"):]))
if err != nil {
return xerrors.Errorf("failed to parse key: %w", err)
}
if goodSet.HasRaw(bk) {
goodHits++
continue
}
nc := cid.NewCidV1(cid.Raw, bk)
deleteCount++
if err := markForRemoval(nc); err != nil {
return fmt.Errorf("failed to remove cid %s: %w", nc, err)
}
if deleteCount%20 == 0 {
fmt.Printf("\rdeleting %d objects (good hits: %d)... ", deleteCount, goodHits)
}
if dupTo != 0 && deleteCount > dupTo {
break
}
}
if b != nil {
if err := b.Commit(); err != nil {
return xerrors.Errorf("failed to commit final batch delete: %w", err)
}
}
gcds, ok := ds.(datastore.GCDatastore)
if ok {
fmt.Println("running datastore gc....")
for i := 0; i < cctx.Int("gc-count"); i++ {
if err := gcds.CollectGarbage(); err != nil {
return xerrors.Errorf("datastore GC failed: %w", err)
}
}
fmt.Println("gc complete!")
}
return nil
},
}
+20 -23
View File
@@ -3,8 +3,6 @@ package main
import (
"fmt"
"github.com/filecoin-project/go-state-types/big"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
@@ -39,31 +37,29 @@ var verifRegCmd = &cli.Command{
}
var verifRegAddVerifierCmd = &cli.Command{
Name: "add-verifier",
Usage: "make a given account a verifier",
ArgsUsage: "<message sender> <new verifier> <allowance>",
Name: "add-verifier",
Usage: "make a given account a verifier",
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 3 {
return fmt.Errorf("must specify three arguments: sender, verifier, and allowance")
}
sender, err := address.NewFromString(cctx.Args().Get(0))
fromk, err := address.NewFromString("t3qfoulel6fy6gn3hjmbhpdpf6fs5aqjb5fkurhtwvgssizq4jey5nw4ptq5up6h7jk7frdvvobv52qzmgjinq")
if err != nil {
return err
}
verifier, err := address.NewFromString(cctx.Args().Get(1))
if cctx.Args().Len() != 2 {
return fmt.Errorf("must specify two arguments: address and allowance")
}
target, err := address.NewFromString(cctx.Args().Get(0))
if err != nil {
return err
}
allowance, err := types.BigFromString(cctx.Args().Get(2))
allowance, err := types.BigFromString(cctx.Args().Get(1))
if err != nil {
return err
}
// TODO: ActorUpgrade: Abstract
params, err := actors.SerializeParams(&verifreg0.AddVerifierParams{Address: verifier, Allowance: allowance})
params, err := actors.SerializeParams(&verifreg0.AddVerifierParams{Address: target, Allowance: allowance})
if err != nil {
return err
}
@@ -75,19 +71,21 @@ var verifRegAddVerifierCmd = &cli.Command{
defer closer()
ctx := lcli.ReqContext(cctx)
vrk, err := api.StateVerifiedRegistryRootKey(ctx, types.EmptyTSK)
msg := &types.Message{
To: verifreg.Address,
From: fromk,
Method: builtin0.MethodsVerifiedRegistry.AddVerifier,
Params: params,
}
smsg, err := api.MpoolPushMessage(ctx, msg, nil)
if err != nil {
return err
}
smsg, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(builtin0.MethodsVerifiedRegistry.AddVerifier), params)
if err != nil {
return err
}
fmt.Printf("message sent, now waiting on cid: %s\n", smsg.Cid())
fmt.Printf("message sent, now waiting on cid: %s\n", smsg)
mwait, err := api.StateWaitMsg(ctx, smsg, build.MessageConfidence)
mwait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence)
if err != nil {
return err
}
@@ -96,7 +94,6 @@ var verifRegAddVerifierCmd = &cli.Command{
return fmt.Errorf("failed to add verifier: %d", mwait.Receipt.ExitCode)
}
//TODO: Internal msg might still have failed
return nil
},
-118
View File
@@ -5,9 +5,6 @@ import (
"os"
"strings"
"github.com/filecoin-project/lotus/build"
builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
"github.com/fatih/color"
"github.com/libp2p/go-libp2p-core/peer"
ma "github.com/multiformats/go-multiaddr"
@@ -35,7 +32,6 @@ var actorCmd = &cli.Command{
actorSetAddrsCmd,
actorWithdrawCmd,
actorSetPeeridCmd,
actorSetOwnerCmd,
actorControl,
},
}
@@ -482,117 +478,3 @@ var actorControlSet = &cli.Command{
return nil
},
}
var actorSetOwnerCmd = &cli.Command{
Name: "set-owner",
Usage: "Set owner address",
ArgsUsage: "[address]",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "really-do-it",
Usage: "Actually send transaction performing the action",
Value: false,
},
},
Action: func(cctx *cli.Context) error {
if !cctx.Bool("really-do-it") {
fmt.Println("Pass --really-do-it to actually execute this action")
return nil
}
if !cctx.Args().Present() {
return fmt.Errorf("must pass address of new owner address")
}
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
api, acloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer acloser()
ctx := lcli.ReqContext(cctx)
na, err := address.NewFromString(cctx.Args().First())
if err != nil {
return err
}
newAddr, err := api.StateLookupID(ctx, na, types.EmptyTSK)
if err != nil {
return err
}
maddr, err := nodeApi.ActorAddress(ctx)
if err != nil {
return err
}
mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
sp, err := actors.SerializeParams(&newAddr)
if err != nil {
return xerrors.Errorf("serializing params: %w", err)
}
smsg, err := api.MpoolPushMessage(ctx, &types.Message{
From: mi.Owner,
To: maddr,
Method: builtin2.MethodsMiner.ChangeOwnerAddress,
Value: big.Zero(),
Params: sp,
}, nil)
if err != nil {
return xerrors.Errorf("mpool push: %w", err)
}
fmt.Println("Propose Message CID:", smsg.Cid())
// wait for it to get mined into a block
wait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence)
if err != nil {
return err
}
// check it executed successfully
if wait.Receipt.ExitCode != 0 {
fmt.Println("Propose owner change failed!")
return err
}
smsg, err = api.MpoolPushMessage(ctx, &types.Message{
From: newAddr,
To: maddr,
Method: builtin2.MethodsMiner.ChangeOwnerAddress,
Value: big.Zero(),
Params: sp,
}, nil)
if err != nil {
return xerrors.Errorf("mpool push: %w", err)
}
fmt.Println("Approve Message CID:", smsg.Cid())
// wait for it to get mined into a block
wait, err = api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence)
if err != nil {
return err
}
// check it executed successfully
if wait.Receipt.ExitCode != 0 {
fmt.Println("Approve owner change failed!")
return err
}
return nil
},
}
-78
View File
@@ -1,78 +0,0 @@
package main
import (
"flag"
"testing"
"time"
"github.com/filecoin-project/lotus/node"
logging "github.com/ipfs/go-log/v2"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api/test"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/lib/lotuslog"
"github.com/filecoin-project/lotus/node/repo"
builder "github.com/filecoin-project/lotus/node/test"
)
func TestMinerAllInfo(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
_ = logging.SetLogLevel("*", "INFO")
policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1)
policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))
_test = true
lotuslog.SetupLogLevels()
logging.SetLogLevel("miner", "ERROR")
logging.SetLogLevel("chainstore", "ERROR")
logging.SetLogLevel("chain", "ERROR")
logging.SetLogLevel("sub", "ERROR")
logging.SetLogLevel("storageminer", "ERROR")
oldDelay := policy.GetPreCommitChallengeDelay()
policy.SetPreCommitChallengeDelay(5)
t.Cleanup(func() {
policy.SetPreCommitChallengeDelay(oldDelay)
})
var n []test.TestNode
var sn []test.TestStorageNode
run := func(t *testing.T) {
app := cli.NewApp()
app.Metadata = map[string]interface{}{
"repoType": repo.StorageMiner,
"testnode-full": n[0],
"testnode-storage": sn[0],
}
build.RunningNodeType = build.NodeMiner
cctx := cli.NewContext(app, flag.NewFlagSet("", flag.ContinueOnError), nil)
require.NoError(t, infoAllCmd.Action(cctx))
}
bp := func(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) {
n, sn = builder.Builder(t, nFull, storage, opts...)
t.Run("pre-info-all", run)
return n, sn
}
test.TestDealFlow(t, bp, time.Second, false, false)
t.Run("post-info-all", run)
}
-14
View File
@@ -1,14 +0,0 @@
package main
import (
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-jsonrpc"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/node/repo"
)
var backupCmd = lcli.BackupCmd(FlagMinerRepo, repo.StorageMiner, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) {
return lcli.GetStorageMinerAPI(cctx)
})
+3 -7
View File
@@ -10,8 +10,6 @@ import (
lcli "github.com/filecoin-project/lotus/cli"
)
var _test = false
var infoAllCmd = &cli.Command{
Name: "all",
Usage: "dump all related miner info",
@@ -152,11 +150,9 @@ var infoAllCmd = &cli.Command{
}
}
if !_test {
fmt.Println("\n#: Goroutines")
if err := lcli.PprofGoroutines.Action(cctx); err != nil {
return err
}
fmt.Println("\n#: Goroutines")
if err := lcli.PprofGoroutines.Action(cctx); err != nil {
return err
}
return nil
-3
View File
@@ -115,9 +115,6 @@ var initCmd = &cli.Command{
Usage: "select which address to send actor creation message from",
},
},
Subcommands: []*cli.Command{
initRestoreCmd,
},
Action: func(cctx *cli.Context) error {
log.Info("Initializing lotus miner")
-274
View File
@@ -1,274 +0,0 @@
package main
import (
"encoding/json"
"io/ioutil"
"os"
"github.com/docker/go-units"
"github.com/ipfs/go-datastore"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/mitchellh/go-homedir"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"gopkg.in/cheggaaa/pb.v1"
"github.com/filecoin-project/go-address"
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/lib/backupds"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/repo"
)
var initRestoreCmd = &cli.Command{
Name: "restore",
Usage: "Initialize a lotus miner repo from a backup",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "nosync",
Usage: "don't check full-node sync status",
},
&cli.StringFlag{
Name: "config",
Usage: "config file (config.toml)",
},
&cli.StringFlag{
Name: "storage-config",
Usage: "storage paths config (storage.json)",
},
},
ArgsUsage: "[backupFile]",
Action: func(cctx *cli.Context) error {
log.Info("Initializing lotus miner using a backup")
if cctx.Args().Len() != 1 {
return xerrors.Errorf("expected 1 argument")
}
log.Info("Trying to connect to full node RPC")
api, closer, err := lcli.GetFullNodeAPI(cctx) // TODO: consider storing full node address in config
if err != nil {
return err
}
defer closer()
log.Info("Checking full node version")
ctx := lcli.ReqContext(cctx)
v, err := api.Version(ctx)
if err != nil {
return err
}
if !v.APIVersion.EqMajorMinor(build.FullAPIVersion) {
return xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", build.FullAPIVersion, v.APIVersion)
}
if !cctx.Bool("nosync") {
if err := lcli.SyncWait(ctx, api); err != nil {
return xerrors.Errorf("sync wait: %w", err)
}
}
bf, err := homedir.Expand(cctx.Args().First())
if err != nil {
return xerrors.Errorf("expand backup file path: %w", err)
}
st, err := os.Stat(bf)
if err != nil {
return xerrors.Errorf("stat backup file (%s): %w", bf, err)
}
f, err := os.Open(bf)
if err != nil {
return xerrors.Errorf("opening backup file: %w", err)
}
defer f.Close() // nolint:errcheck
log.Info("Checking if repo exists")
repoPath := cctx.String(FlagMinerRepo)
r, err := repo.NewFS(repoPath)
if err != nil {
return err
}
ok, err := r.Exists()
if err != nil {
return err
}
if ok {
return xerrors.Errorf("repo at '%s' is already initialized", cctx.String(FlagMinerRepo))
}
log.Info("Initializing repo")
if err := r.Init(repo.StorageMiner); err != nil {
return err
}
lr, err := r.Lock(repo.StorageMiner)
if err != nil {
return err
}
defer lr.Close() //nolint:errcheck
if cctx.IsSet("config") {
log.Info("Restoring config")
cf, err := homedir.Expand(cctx.String("config"))
if err != nil {
return xerrors.Errorf("expanding config path: %w", err)
}
_, err = os.Stat(cf)
if err != nil {
return xerrors.Errorf("stat config file (%s): %w", cf, err)
}
var cerr error
err = lr.SetConfig(func(raw interface{}) {
rcfg, ok := raw.(*config.StorageMiner)
if !ok {
cerr = xerrors.New("expected miner config")
return
}
ff, err := config.FromFile(cf, rcfg)
if err != nil {
cerr = xerrors.Errorf("loading config: %w", err)
return
}
*rcfg = *ff.(*config.StorageMiner)
})
if cerr != nil {
return cerr
}
if err != nil {
return xerrors.Errorf("setting config: %w", err)
}
} else {
log.Warn("--config NOT SET, WILL USE DEFAULT VALUES")
}
if cctx.IsSet("storage-config") {
log.Info("Restoring storage path config")
cf, err := homedir.Expand(cctx.String("storage-config"))
if err != nil {
return xerrors.Errorf("expanding storage config path: %w", err)
}
cfb, err := ioutil.ReadFile(cf)
if err != nil {
return xerrors.Errorf("reading storage config: %w", err)
}
var cerr error
err = lr.SetStorage(func(scfg *stores.StorageConfig) {
cerr = json.Unmarshal(cfb, scfg)
})
if cerr != nil {
return xerrors.Errorf("unmarshalling storage config: %w", cerr)
}
if err != nil {
return xerrors.Errorf("setting storage config: %w", err)
}
} else {
log.Warn("--storage-config NOT SET. NO SECTOR PATHS WILL BE CONFIGURED")
}
log.Info("Restoring metadata backup")
mds, err := lr.Datastore("/metadata")
if err != nil {
return err
}
bar := pb.New64(st.Size())
br := bar.NewProxyReader(f)
bar.ShowTimeLeft = true
bar.ShowPercent = true
bar.ShowSpeed = true
bar.Units = pb.U_BYTES
bar.Start()
err = backupds.RestoreInto(br, mds)
bar.Finish()
if err != nil {
return xerrors.Errorf("restoring metadata: %w", err)
}
log.Info("Checking actor metadata")
abytes, err := mds.Get(datastore.NewKey("miner-address"))
if err != nil {
return xerrors.Errorf("getting actor address from metadata datastore: %w", err)
}
maddr, err := address.NewFromBytes(abytes)
if err != nil {
return xerrors.Errorf("parsing actor address: %w", err)
}
log.Info("ACTOR ADDRESS: ", maddr.String())
mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("getting miner info: %w", err)
}
log.Info("SECTOR SIZE: ", units.BytesSize(float64(mi.SectorSize)))
wk, err := api.StateAccountKey(ctx, mi.Worker, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("resolving worker key: %w", err)
}
has, err := api.WalletHas(ctx, wk)
if err != nil {
return xerrors.Errorf("checking worker address: %w", err)
}
if !has {
return xerrors.Errorf("worker address %s for miner actor %s not present in full node wallet", mi.Worker, maddr)
}
log.Info("Checking proof parameters")
if err := paramfetch.GetParams(ctx, build.ParametersJSON(), uint64(mi.SectorSize)); err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
}
log.Info("Initializing libp2p identity")
p2pSk, err := makeHostKey(lr)
if err != nil {
return xerrors.Errorf("make host key: %w", err)
}
peerid, err := peer.IDFromPrivateKey(p2pSk)
if err != nil {
return xerrors.Errorf("peer ID from private key: %w", err)
}
log.Info("Configuring miner actor")
if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil {
return err
}
return nil
},
}
-1
View File
@@ -35,7 +35,6 @@ func main() {
runCmd,
stopCmd,
configCmd,
backupCmd,
lcli.WithCategory("chain", actorCmd),
lcli.WithCategory("chain", infoCmd),
lcli.WithCategory("market", storageDealsCmd),
+7 -1
View File
@@ -164,6 +164,12 @@ var setAskCmd = &cli.Command{
Usage: "Set the price of the ask for verified deals (specified as attoFIL / GiB / Epoch) to `PRICE`",
Required: true,
},
&cli.StringFlag{
Name: "duration",
Usage: "Set duration of ask (a quantity of time after which the ask expires) `DURATION`",
DefaultText: "720h0m0s",
Value: "720h0m0s",
},
&cli.StringFlag{
Name: "min-piece-size",
Usage: "Set minimum piece size (w/bit-padding, in bytes) in ask to `SIZE`",
@@ -188,7 +194,7 @@ var setAskCmd = &cli.Command{
pri := types.NewInt(cctx.Uint64("price"))
vpri := types.NewInt(cctx.Uint64("verified-price"))
dur, err := time.ParseDuration("720h0m0s")
dur, err := time.ParseDuration(cctx.String("duration"))
if err != nil {
return xerrors.Errorf("cannot parse duration: %w", err)
}
+19 -92
View File
@@ -5,22 +5,19 @@ import (
"os"
"sort"
"strconv"
"text/tabwriter"
"time"
"github.com/docker/go-units"
"github.com/fatih/color"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/tablewriter"
lcli "github.com/filecoin-project/lotus/cli"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
@@ -147,19 +144,8 @@ var sectorsListCmd = &cli.Command{
Name: "show-removed",
Usage: "show removed sectors",
},
&cli.BoolFlag{
Name: "color",
Aliases: []string{"c"},
Value: true,
},
&cli.BoolFlag{
Name: "fast",
Usage: "don't show on-chain info for better performance",
},
},
Action: func(cctx *cli.Context) error {
color.NoColor = !cctx.Bool("color")
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
@@ -184,12 +170,7 @@ var sectorsListCmd = &cli.Command{
return err
}
head, err := fullApi.ChainHead(ctx)
if err != nil {
return err
}
activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, head.Key())
activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
@@ -198,7 +179,7 @@ var sectorsListCmd = &cli.Command{
activeIDs[info.SectorNumber] = struct{}{}
}
sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, head.Key())
sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, types.EmptyTSK)
if err != nil {
return err
}
@@ -211,26 +192,12 @@ var sectorsListCmd = &cli.Command{
return list[i] < list[j]
})
tw := tablewriter.New(
tablewriter.Col("ID"),
tablewriter.Col("State"),
tablewriter.Col("OnChain"),
tablewriter.Col("Active"),
tablewriter.Col("Expiration"),
tablewriter.Col("Deals"),
tablewriter.Col("DealWeight"),
tablewriter.NewLineCol("Error"),
tablewriter.NewLineCol("EarlyExpiration"))
fast := cctx.Bool("fast")
w := tabwriter.NewWriter(os.Stdout, 8, 4, 1, ' ', 0)
for _, s := range list {
st, err := nodeApi.SectorsStatus(ctx, s, !fast)
st, err := nodeApi.SectorsStatus(ctx, s, false)
if err != nil {
tw.Write(map[string]interface{}{
"ID": s,
"Error": err,
})
fmt.Fprintf(w, "%d:\tError: %s\n", s, err)
continue
}
@@ -238,60 +205,20 @@ var sectorsListCmd = &cli.Command{
_, inSSet := commitedIDs[s]
_, inASet := activeIDs[s]
dw := .0
if st.Expiration-st.Activation > 0 {
dw = float64(big.Div(st.DealWeight, big.NewInt(int64(st.Expiration-st.Activation))).Uint64())
}
var deals int
for _, deal := range st.Deals {
if deal != 0 {
deals++
}
}
exp := st.Expiration
if st.OnTime > 0 && st.OnTime < exp {
exp = st.OnTime // Can be different when the sector was CC upgraded
}
m := map[string]interface{}{
"ID": s,
"State": color.New(stateOrder[sealing.SectorState(st.State)].col).Sprint(st.State),
"OnChain": yesno(inSSet),
"Active": yesno(inASet),
}
if deals > 0 {
m["Deals"] = color.GreenString("%d", deals)
} else {
m["Deals"] = color.BlueString("CC")
if st.ToUpgrade {
m["Deals"] = color.CyanString("CC(upgrade)")
}
}
if !fast {
if !inSSet {
m["Expiration"] = "n/a"
} else {
m["Expiration"] = lcli.EpochTime(head.Height(), exp)
if !fast && deals > 0 {
m["DealWeight"] = units.BytesSize(dw)
}
if st.Early > 0 {
m["EarlyExpiration"] = color.YellowString(lcli.EpochTime(head.Height(), st.Early))
}
}
}
tw.Write(m)
_, _ = fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n",
s,
st.State,
yesno(inSSet),
yesno(inASet),
st.Ticket.Epoch,
st.Seed.Epoch,
st.Deals,
st.ToUpgrade,
)
}
}
return tw.Flush(os.Stdout)
return w.Flush()
},
}
@@ -520,7 +447,7 @@ var sectorsUpdateCmd = &cli.Command{
func yesno(b bool) string {
if b {
return color.GreenString("YES")
return "YES"
}
return color.RedString("NO")
return "NO"
}
-14
View File
@@ -1,14 +0,0 @@
package main
import (
"github.com/urfave/cli/v2"
"github.com/filecoin-project/go-jsonrpc"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/node/repo"
)
var backupCmd = lcli.BackupCmd("repo", repo.FullNode, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) {
return lcli.GetFullNodeAPI(cctx)
})
-1
View File
@@ -22,7 +22,6 @@ func main() {
local := []*cli.Command{
DaemonCmd,
backupCmd,
}
if AdvanceBlockCmd != nil {
local = append(local, AdvanceBlockCmd)
+1 -1
View File
@@ -167,7 +167,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP
// dummy state manager; only to reference the GetNetworkVersion method,
// which does not depend on state.
sm := stmgr.NewStateManager(nil)
sm := new(stmgr.StateManager)
vmOpts := &vm.VMOpts{
StateBase: params.Preroot,
+1 -6
View File
@@ -233,12 +233,7 @@ func writeStateToTempCAR(bs blockstore.Blockstore, roots ...cid.Cid) (string, er
if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
continue
}
// ignore things we don't have, the state tree is incomplete.
if has, err := bs.Has(link.Cid); err != nil {
return nil, err
} else if has {
out = append(out, link)
}
out = append(out, link)
}
return out, nil
}
File diff suppressed because it is too large Load Diff
Vendored
+1 -1
Submodule extern/oni updated: dbee44e4f9...9a0d5cd739
+52 -76
View File
@@ -10,38 +10,14 @@ type Resources struct {
MinMemory uint64 // What Must be in RAM for decent perf
MaxMemory uint64 // Memory required (swap + ram)
MaxParallelism int // -1 = multithread
CanGPU bool
Threads int // -1 = multithread
CanGPU bool
BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads)
}
/*
Percent of threads to allocate to parallel tasks
12 * 0.92 = 11
16 * 0.92 = 14
24 * 0.92 = 22
32 * 0.92 = 29
64 * 0.92 = 58
128 * 0.92 = 117
*/
var ParallelNum uint64 = 92
var ParallelDenom uint64 = 100
// TODO: Take NUMA into account
func (r Resources) Threads(wcpus uint64) uint64 {
if r.MaxParallelism == -1 {
n := (wcpus * ParallelNum) / ParallelDenom
if n == 0 {
return wcpus
}
return n
}
return uint64(r.MaxParallelism)
func (r Resources) MultiThread() bool {
return r.Threads == -1
}
var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{
@@ -50,7 +26,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 8 << 30,
MinMemory: 8 << 30,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 1 << 30,
},
@@ -58,7 +34,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 4 << 30,
MinMemory: 4 << 30,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 1 << 30,
},
@@ -66,7 +42,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 30,
MinMemory: 1 << 30,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 1 << 30,
},
@@ -74,7 +50,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 2 << 10,
MinMemory: 2 << 10,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 2 << 10,
},
@@ -82,7 +58,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 8 << 20,
MinMemory: 8 << 20,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 8 << 20,
},
@@ -92,7 +68,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 128 << 30,
MinMemory: 112 << 30,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 10 << 20,
},
@@ -100,7 +76,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 64 << 30,
MinMemory: 56 << 30,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 10 << 20,
},
@@ -108,7 +84,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 30,
MinMemory: 768 << 20,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 1 << 20,
},
@@ -116,7 +92,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 2 << 10,
MinMemory: 2 << 10,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 2 << 10,
},
@@ -124,35 +100,35 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 8 << 20,
MinMemory: 8 << 20,
MaxParallelism: 1,
Threads: 1,
BaseMinMemory: 8 << 20,
},
},
sealtasks.TTPreCommit2: {
abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
MaxMemory: 30 << 30,
MinMemory: 30 << 30,
MaxMemory: 64 << 30,
MinMemory: 64 << 30,
MaxParallelism: -1,
CanGPU: true,
Threads: -1,
CanGPU: true,
BaseMinMemory: 1 << 30,
BaseMinMemory: 60 << 30,
},
abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
MaxMemory: 15 << 30,
MinMemory: 15 << 30,
MaxMemory: 32 << 30,
MinMemory: 32 << 30,
MaxParallelism: -1,
CanGPU: true,
Threads: -1,
CanGPU: true,
BaseMinMemory: 1 << 30,
BaseMinMemory: 30 << 30,
},
abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
MaxMemory: 3 << 29, // 1.5G
MinMemory: 1 << 30,
MaxParallelism: -1,
Threads: -1,
BaseMinMemory: 1 << 30,
},
@@ -160,7 +136,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 2 << 10,
MinMemory: 2 << 10,
MaxParallelism: -1,
Threads: -1,
BaseMinMemory: 2 << 10,
},
@@ -168,7 +144,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 8 << 20,
MinMemory: 8 << 20,
MaxParallelism: -1,
Threads: -1,
BaseMinMemory: 8 << 20,
},
@@ -178,7 +154,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 30,
MinMemory: 1 << 30,
MaxParallelism: 0,
Threads: 0,
BaseMinMemory: 1 << 30,
},
@@ -186,7 +162,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 30,
MinMemory: 1 << 30,
MaxParallelism: 0,
Threads: 0,
BaseMinMemory: 1 << 30,
},
@@ -194,7 +170,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 30,
MinMemory: 1 << 30,
MaxParallelism: 0,
Threads: 0,
BaseMinMemory: 1 << 30,
},
@@ -202,7 +178,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 2 << 10,
MinMemory: 2 << 10,
MaxParallelism: 0,
Threads: 0,
BaseMinMemory: 2 << 10,
},
@@ -210,7 +186,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 8 << 20,
MinMemory: 8 << 20,
MaxParallelism: 0,
Threads: 0,
BaseMinMemory: 8 << 20,
},
@@ -220,8 +196,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 190 << 30, // TODO: Confirm
MinMemory: 60 << 30,
MaxParallelism: -1,
CanGPU: true,
Threads: -1,
CanGPU: true,
BaseMinMemory: 64 << 30, // params
},
@@ -229,8 +205,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory
MinMemory: 30 << 30,
MaxParallelism: -1,
CanGPU: true,
Threads: -1,
CanGPU: true,
BaseMinMemory: 32 << 30, // params
},
@@ -238,8 +214,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 3 << 29, // 1.5G
MinMemory: 1 << 30,
MaxParallelism: 1, // This is fine
CanGPU: true,
Threads: 1, // This is fine
CanGPU: true,
BaseMinMemory: 10 << 30,
},
@@ -247,8 +223,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 2 << 10,
MinMemory: 2 << 10,
MaxParallelism: 1,
CanGPU: true,
Threads: 1,
CanGPU: true,
BaseMinMemory: 2 << 10,
},
@@ -256,8 +232,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 8 << 20,
MinMemory: 8 << 20,
MaxParallelism: 1,
CanGPU: true,
Threads: 1,
CanGPU: true,
BaseMinMemory: 8 << 20,
},
@@ -267,8 +243,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 20,
MinMemory: 1 << 20,
MaxParallelism: 0,
CanGPU: false,
Threads: 0,
CanGPU: false,
BaseMinMemory: 0,
},
@@ -276,8 +252,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 20,
MinMemory: 1 << 20,
MaxParallelism: 0,
CanGPU: false,
Threads: 0,
CanGPU: false,
BaseMinMemory: 0,
},
@@ -285,8 +261,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 20,
MinMemory: 1 << 20,
MaxParallelism: 0,
CanGPU: false,
Threads: 0,
CanGPU: false,
BaseMinMemory: 0,
},
@@ -294,8 +270,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 20,
MinMemory: 1 << 20,
MaxParallelism: 0,
CanGPU: false,
Threads: 0,
CanGPU: false,
BaseMinMemory: 0,
},
@@ -303,8 +279,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
MaxMemory: 1 << 20,
MinMemory: 1 << 20,
MaxParallelism: 0,
CanGPU: false,
Threads: 0,
CanGPU: false,
BaseMinMemory: 0,
},
+22 -5
View File
@@ -28,7 +28,12 @@ func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResource
func (a *activeResources) add(wr storiface.WorkerResources, r Resources) {
a.gpuUsed = r.CanGPU
a.cpuUse += r.Threads(wr.CPUs)
if r.MultiThread() {
a.cpuUse += wr.CPUs
} else {
a.cpuUse += uint64(r.Threads)
}
a.memUsedMin += r.MinMemory
a.memUsedMax += r.MaxMemory
}
@@ -37,7 +42,12 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) {
if r.CanGPU {
a.gpuUsed = false
}
a.cpuUse -= r.Threads(wr.CPUs)
if r.MultiThread() {
a.cpuUse -= wr.CPUs
} else {
a.cpuUse -= uint64(r.Threads)
}
a.memUsedMin -= r.MinMemory
a.memUsedMax -= r.MaxMemory
}
@@ -58,9 +68,16 @@ func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, call
return false
}
if a.cpuUse+needRes.Threads(res.CPUs) > res.CPUs {
log.Debugf("sched: not scheduling on worker %d for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads(res.CPUs), a.cpuUse, res.CPUs)
return false
if needRes.MultiThread() {
if a.cpuUse > 0 {
log.Debugf("sched: not scheduling on worker %d for %s; multicore process needs %d threads, %d in use, target %d", wid, caller, res.CPUs, a.cpuUse, res.CPUs)
return false
}
} else {
if a.cpuUse+uint64(needRes.Threads) > res.CPUs {
log.Debugf("sched: not scheduling on worker %d for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads, a.cpuUse, res.CPUs)
return false
}
}
if len(res.GPUs) > 0 && needRes.CanGPU {
-3
View File
@@ -290,9 +290,6 @@ func TestSched(t *testing.T) {
}
testFunc := func(workers []workerSpec, tasks []task) func(t *testing.T) {
ParallelNum = 1
ParallelDenom = 1
return func(t *testing.T) {
index := stores.NewIndex()
+2 -6
View File
@@ -23,8 +23,7 @@ require (
github.com/fatih/color v1.8.0
github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d
github.com/filecoin-project/go-address v0.0.4
github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect
github.com/filecoin-project/go-bitfield v0.2.1
github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
github.com/filecoin-project/go-data-transfer v0.6.7
@@ -39,7 +38,7 @@ require (
github.com/filecoin-project/go-statestore v0.1.0
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
github.com/filecoin-project/specs-actors v0.9.12
github.com/filecoin-project/specs-actors/v2 v2.0.1
github.com/filecoin-project/specs-actors/v2 v2.0.0-20201001041506-c7ff44a3ce9b
github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796
github.com/filecoin-project/test-vectors/schema v0.0.3
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
@@ -48,12 +47,10 @@ require (
github.com/google/uuid v1.1.1
github.com/gorilla/mux v1.7.4
github.com/gorilla/websocket v1.4.2
github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e
github.com/hashicorp/go-multierror v1.1.0
github.com/hashicorp/golang-lru v0.5.4
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d
github.com/ipfs/bbloom v0.0.4
github.com/ipfs/go-bitswap v0.2.20
github.com/ipfs/go-block-format v0.0.2
github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834
@@ -112,7 +109,6 @@ require (
github.com/multiformats/go-multibase v0.0.3
github.com/multiformats/go-multihash v0.0.14
github.com/opentracing/opentracing-go v1.2.0
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
github.com/raulk/clock v1.1.0
github.com/stretchr/testify v1.6.1
github.com/supranational/blst v0.1.1
+5 -9
View File
@@ -228,12 +228,10 @@ github.com/filecoin-project/go-address v0.0.4 h1:gSNMv0qWwH16fGQs7ycOUrDjY6YCSsg
github.com/filecoin-project/go-address v0.0.4/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM=
github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs=
github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 h1:pIuR0dnMD0i+as8wNnjjHyQrnhP5O5bmba/lmgQeRgU=
github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g=
github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q=
github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
github.com/filecoin-project/go-bitfield v0.2.1 h1:S6Uuqcspqu81sWJ0He4OAfFLm1tSwPdVjtKTkl5m/xQ=
github.com/filecoin-project/go-bitfield v0.2.1/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g=
github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
@@ -276,8 +274,8 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07
github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU=
github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk=
github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
github.com/filecoin-project/specs-actors/v2 v2.0.1 h1:bf08x6tqCDfClzrv2q/rmt/A/UbBOy1KgaoogQEcLhU=
github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY=
github.com/filecoin-project/specs-actors/v2 v2.0.0-20201001041506-c7ff44a3ce9b h1:8JRxR0rKdiuCHIDKi7Zbs+/jiygr6eXEX47QXhjX+kA=
github.com/filecoin-project/specs-actors/v2 v2.0.0-20201001041506-c7ff44a3ce9b/go.mod h1:52FuQUNDXq2WDg+6+UOhkqBuNc2e62h9BCIB67Bluxg=
github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk=
github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU=
@@ -423,8 +421,6 @@ github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE=
github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 h1:BpJ2o0OR5FV7vrkDYfXYVJQeMNWa8RhklZOpW2ITAIQ=
github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE=
github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1 h1:F9k+7wv5OIk1zcq23QpdiL0hfDuXPjuOmMNaC6fgQ0Q=
github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8=
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e h1:3YKHER4nmd7b5qy5t0GWDTwSn4OyRgfAXSmo6VnryBY=
@@ -1402,7 +1398,6 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:X
github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg=
github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
github.com/whyrusleeping/cbor-gen v0.0.0-20200723185710-6a3894a6352b/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
@@ -1528,6 +1523,7 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc=
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
-189
View File
@@ -1,189 +0,0 @@
package backupds
import (
"crypto/sha256"
"io"
"sync"
logging "github.com/ipfs/go-log/v2"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
)
var log = logging.Logger("backupds")
type Datastore struct {
child datastore.Batching
backupLk sync.RWMutex
}
func Wrap(child datastore.Batching) *Datastore {
return &Datastore{
child: child,
}
}
// Writes a datastore dump into the provided writer as
// [array(*) of [key, value] tuples, checksum]
func (d *Datastore) Backup(out io.Writer) error {
scratch := make([]byte, 9)
if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajArray, 2); err != nil {
return xerrors.Errorf("writing tuple header: %w", err)
}
hasher := sha256.New()
hout := io.MultiWriter(hasher, out)
// write KVs
{
// write indefinite length array header
if _, err := hout.Write([]byte{0x9f}); err != nil {
return xerrors.Errorf("writing header: %w", err)
}
d.backupLk.Lock()
defer d.backupLk.Unlock()
log.Info("Starting datastore backup")
defer log.Info("Datastore backup done")
qr, err := d.child.Query(query.Query{})
if err != nil {
return xerrors.Errorf("query: %w", err)
}
defer func() {
if err := qr.Close(); err != nil {
log.Errorf("query close error: %+v", err)
return
}
}()
for result := range qr.Next() {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajArray, 2); err != nil {
return xerrors.Errorf("writing tuple header: %w", err)
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajByteString, uint64(len([]byte(result.Key)))); err != nil {
return xerrors.Errorf("writing key header: %w", err)
}
if _, err := hout.Write([]byte(result.Key)[:]); err != nil {
return xerrors.Errorf("writing key: %w", err)
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajByteString, uint64(len(result.Value))); err != nil {
return xerrors.Errorf("writing value header: %w", err)
}
if _, err := hout.Write(result.Value[:]); err != nil {
return xerrors.Errorf("writing value: %w", err)
}
}
// array break
if _, err := hout.Write([]byte{0xff}); err != nil {
return xerrors.Errorf("writing array 'break': %w", err)
}
}
// Write the checksum
{
sum := hasher.Sum(nil)
if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajByteString, uint64(len(sum))); err != nil {
return xerrors.Errorf("writing checksum header: %w", err)
}
if _, err := hout.Write(sum[:]); err != nil {
return xerrors.Errorf("writing checksum: %w", err)
}
}
return nil
}
// proxy
func (d *Datastore) Get(key datastore.Key) (value []byte, err error) {
return d.child.Get(key)
}
func (d *Datastore) Has(key datastore.Key) (exists bool, err error) {
return d.child.Has(key)
}
func (d *Datastore) GetSize(key datastore.Key) (size int, err error) {
return d.child.GetSize(key)
}
func (d *Datastore) Query(q query.Query) (query.Results, error) {
return d.child.Query(q)
}
func (d *Datastore) Put(key datastore.Key, value []byte) error {
d.backupLk.RLock()
defer d.backupLk.RUnlock()
return d.child.Put(key, value)
}
func (d *Datastore) Delete(key datastore.Key) error {
d.backupLk.RLock()
defer d.backupLk.RUnlock()
return d.child.Delete(key)
}
func (d *Datastore) Sync(prefix datastore.Key) error {
d.backupLk.RLock()
defer d.backupLk.RUnlock()
return d.child.Sync(prefix)
}
func (d *Datastore) Close() error {
d.backupLk.RLock()
defer d.backupLk.RUnlock()
return d.child.Close()
}
func (d *Datastore) Batch() (datastore.Batch, error) {
b, err := d.child.Batch()
if err != nil {
return nil, err
}
return &bbatch{
b: b,
rlk: d.backupLk.RLocker(),
}, nil
}
type bbatch struct {
b datastore.Batch
rlk sync.Locker
}
func (b *bbatch) Put(key datastore.Key, value []byte) error {
return b.b.Put(key, value)
}
func (b *bbatch) Delete(key datastore.Key) error {
return b.b.Delete(key)
}
func (b *bbatch) Commit() error {
b.rlk.Lock()
defer b.rlk.Unlock()
return b.b.Commit()
}
var _ datastore.Batch = &bbatch{}
var _ datastore.Batching = &Datastore{}
-100
View File
@@ -1,100 +0,0 @@
package backupds
import (
"bytes"
"crypto/sha256"
"io"
"github.com/ipfs/go-datastore"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
)
func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) error {
scratch := make([]byte, 9)
if _, err := r.Read(scratch[:1]); err != nil {
return xerrors.Errorf("reading array header: %w", err)
}
if scratch[0] != 0x82 {
return xerrors.Errorf("expected array(2) header byte 0x82, got %x", scratch[0])
}
hasher := sha256.New()
hr := io.TeeReader(r, hasher)
if _, err := hr.Read(scratch[:1]); err != nil {
return xerrors.Errorf("reading array header: %w", err)
}
if scratch[0] != 0x9f {
return xerrors.Errorf("expected indefinite length array header byte 0x9f, got %x", scratch[0])
}
for {
if _, err := hr.Read(scratch[:1]); err != nil {
return xerrors.Errorf("reading tuple header: %w", err)
}
if scratch[0] == 0xff {
break
}
if scratch[0] != 0x82 {
return xerrors.Errorf("expected array(2) header 0x82, got %x", scratch[0])
}
keyb, err := cbg.ReadByteArray(hr, 1<<40)
if err != nil {
return xerrors.Errorf("reading key: %w", err)
}
key := datastore.NewKey(string(keyb))
value, err := cbg.ReadByteArray(hr, 1<<40)
if err != nil {
return xerrors.Errorf("reading value: %w", err)
}
if err := cb(key, value); err != nil {
return err
}
}
sum := hasher.Sum(nil)
expSum, err := cbg.ReadByteArray(r, 32)
if err != nil {
return xerrors.Errorf("reading expected checksum: %w", err)
}
if !bytes.Equal(sum, expSum) {
return xerrors.Errorf("checksum didn't match; expected %x, got %x", expSum, sum)
}
return nil
}
func RestoreInto(r io.Reader, dest datastore.Batching) error {
batch, err := dest.Batch()
if err != nil {
return xerrors.Errorf("creating batch: %w", err)
}
err = ReadBackup(r, func(key datastore.Key, value []byte) error {
if err := batch.Put(key, value); err != nil {
return xerrors.Errorf("put key: %w", err)
}
return nil
})
if err != nil {
return xerrors.Errorf("reading backup: %w", err)
}
if err := batch.Commit(); err != nil {
return xerrors.Errorf("committing batch: %w", err)
}
return nil
}
+2 -13
View File
@@ -12,7 +12,6 @@ import (
type Column struct {
Name string
SeparateLine bool
Lines int
}
type TableWriter struct {
@@ -51,7 +50,6 @@ cloop:
for i, column := range w.cols {
if column.Name == col {
byColID[i] = fmt.Sprint(val)
w.cols[i].Lines++
continue cloop
}
}
@@ -60,7 +58,6 @@ cloop:
w.cols = append(w.cols, Column{
Name: col,
SeparateLine: false,
Lines: 1,
})
}
@@ -80,11 +77,7 @@ func (w *TableWriter) Flush(out io.Writer) error {
w.rows = append([]map[int]string{header}, w.rows...)
for col, c := range w.cols {
if c.Lines == 0 {
continue
}
for col := range w.cols {
for _, row := range w.rows {
val, found := row[col]
if !found {
@@ -101,13 +94,9 @@ func (w *TableWriter) Flush(out io.Writer) error {
cols := make([]string, len(w.cols))
for ci, col := range w.cols {
if col.Lines == 0 {
continue
}
e, _ := row[ci]
pad := colLengths[ci] - cliStringLength(e) + 2
if !col.SeparateLine && col.Lines > 0 {
if !col.SeparateLine {
e = e + strings.Repeat(" ", pad)
if _, err := fmt.Fprint(out, e); err != nil {
return err
+10 -20
View File
@@ -25,15 +25,15 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/events/state"
"github.com/filecoin-project/lotus/chain/types"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
@@ -50,24 +50,14 @@ type ProviderNodeAdapter struct {
secb *sectorblocks.SectorBlocks
ev *events.Events
publishSpec, addBalanceSpec *api.MessageSendSpec
}
func NewProviderNodeAdapter(fc *config.MinerFeeConfig) func(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode {
return func(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode {
na := &ProviderNodeAdapter{
FullNode: full,
dag: dag,
secb: secb,
ev: events.NewEvents(context.TODO(), full),
}
if fc != nil {
na.publishSpec = &api.MessageSendSpec{MaxFee: abi.TokenAmount(fc.MaxPublishDealsFee)}
na.addBalanceSpec = &api.MessageSendSpec{MaxFee: abi.TokenAmount(fc.MaxMarketBalanceAddFee)}
}
return na
func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode {
return &ProviderNodeAdapter{
FullNode: full,
dag: dag,
secb: secb,
ev: events.NewEvents(context.TODO(), full),
}
}
@@ -94,7 +84,7 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark
Value: types.NewInt(0),
Method: builtin0.MethodsMarket.PublishStorageDeals,
Params: params,
}, n.publishSpec)
}, nil)
if err != nil {
return cid.Undef, err
}
@@ -193,7 +183,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address
From: addr,
Value: amount,
Method: builtin0.MethodsMarket.AddBalance,
}, n.addBalanceSpec)
}, nil)
if err != nil {
return cid.Undef, err
}
+1 -1
View File
@@ -362,7 +362,7 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg,
if mbi == nil {
return nil, nil
}
if !mbi.EligibleForMining {
if !mbi.HasMinPower {
// slashed or just have no power yet
return nil, nil
}
+2 -5
View File
@@ -259,8 +259,7 @@ func Online() Option {
Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier),
Override(new(vm.SyscallBuilder), vm.Syscalls),
Override(new(*store.ChainStore), modules.ChainStore),
Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()),
Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule),
Override(new(*stmgr.StateManager), stmgr.NewStateManager),
Override(new(*wallet.Wallet), wallet.NewWallet),
Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner),
@@ -345,7 +344,7 @@ func Online() Option {
Override(new(dtypes.DealFilter), modules.BasicDealFilter(nil)),
Override(new(modules.ProviderDealFunds), modules.NewProviderDealFunds),
Override(new(storagemarket.StorageProvider), modules.StorageProvider),
Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(nil)),
Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter),
Override(HandleRetrievalKey, modules.HandleRetrieval),
Override(GetParamsKey, modules.GetParams),
Override(HandleDealsKey, modules.HandleDeals),
@@ -464,8 +463,6 @@ func ConfigStorageMiner(c interface{}) Option {
Override(new(dtypes.DealFilter), modules.BasicDealFilter(dealfilter.CliDealFilter(cfg.Dealmaking.Filter))),
),
Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees)),
Override(new(sectorstorage.SealerConfig), cfg.Storage),
Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)),
)
+8 -12
View File
@@ -61,11 +61,9 @@ type SealingConfig struct {
}
type MinerFeeConfig struct {
MaxPreCommitGasFee types.FIL
MaxCommitGasFee types.FIL
MaxWindowPoStGasFee types.FIL
MaxPublishDealsFee types.FIL
MaxMarketBalanceAddFee types.FIL
MaxPreCommitGasFee types.FIL
MaxCommitGasFee types.FIL
MaxWindowPoStGasFee types.FIL
}
// API contains configs for API endpoint
@@ -149,7 +147,7 @@ func DefaultStorageMiner() *StorageMiner {
MaxWaitDealsSectors: 2, // 64G with 32G sectors
MaxSealingSectors: 0,
MaxSealingSectorsForDeals: 0,
WaitDealsDelay: Duration(time.Hour * 6),
WaitDealsDelay: Duration(time.Hour),
},
Storage: sectorstorage.SealerConfig{
@@ -171,15 +169,13 @@ func DefaultStorageMiner() *StorageMiner {
ConsiderOfflineRetrievalDeals: true,
PieceCidBlocklist: []cid.Cid{},
// TODO: It'd be nice to set this based on sector size
ExpectedSealDuration: Duration(time.Hour * 24),
ExpectedSealDuration: Duration(time.Hour * 12),
},
Fees: MinerFeeConfig{
MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05
MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))),
MaxWindowPoStGasFee: types.FIL(types.FromFil(50)),
MaxPublishDealsFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(33))), // 0.03ish
MaxMarketBalanceAddFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(100))), // 0.01
MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05
MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))),
MaxWindowPoStGasFee: types.FIL(types.FromFil(50)),
},
}
cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"
-67
View File
@@ -1,67 +0,0 @@
package impl
import (
"os"
"path/filepath"
"strings"
"github.com/mitchellh/go-homedir"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/backupds"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func backup(mds dtypes.MetadataDS, fpath string) error {
bb, ok := os.LookupEnv("LOTUS_BACKUP_BASE_PATH")
if !ok {
return xerrors.Errorf("LOTUS_BACKUP_BASE_PATH env var not set")
}
bds, ok := mds.(*backupds.Datastore)
if !ok {
return xerrors.Errorf("expected a backup datastore")
}
bb, err := homedir.Expand(bb)
if err != nil {
return xerrors.Errorf("expanding base path: %w", err)
}
bb, err = filepath.Abs(bb)
if err != nil {
return xerrors.Errorf("getting absolute base path: %w", err)
}
fpath, err = homedir.Expand(fpath)
if err != nil {
return xerrors.Errorf("expanding file path: %w", err)
}
fpath, err = filepath.Abs(fpath)
if err != nil {
return xerrors.Errorf("getting absolute file path: %w", err)
}
if !strings.HasPrefix(fpath, bb) {
return xerrors.Errorf("backup file name (%s) must be inside base path (%s)", fpath, bb)
}
out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return xerrors.Errorf("open %s: %w", fpath, err)
}
if err := bds.Backup(out); err != nil {
if cerr := out.Close(); cerr != nil {
log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err)
}
return xerrors.Errorf("backup error: %w", err)
}
if err := out.Close(); err != nil {
return xerrors.Errorf("closing backup file: %w", err)
}
return nil
}
+3 -12
View File
@@ -59,7 +59,7 @@ import (
var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31)
const dealStartBufferHours uint64 = 49
const dealStartBufferHours uint64 = 24
type API struct {
fx.In
@@ -116,13 +116,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams)
}
}
}
walletKey, err := a.StateAPI.StateManager.ResolveToKeyAddress(ctx, params.Wallet, nil)
if err != nil {
return nil, xerrors.Errorf("failed resolving params.Wallet addr: %w", params.Wallet)
}
exist, err := a.WalletHas(ctx, walletKey)
exist, err := a.WalletHas(ctx, params.Wallet)
if err != nil {
return nil, xerrors.Errorf("failed getting addr from wallet: %w", params.Wallet)
}
@@ -159,7 +153,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams)
}
blocksPerHour := 60 * 60 / build.BlockDelaySecs
dealStart = ts.Height() + abi.ChainEpoch(dealStartBufferHours*blocksPerHour) // TODO: Get this from storage ask
dealStart = ts.Height() + abi.ChainEpoch(dealStartBufferHours*blocksPerHour)
}
result, err := a.SMDealClient.ProposeStorageDeal(ctx, storagemarket.ProposeStorageDealParams{
@@ -205,7 +199,6 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
Duration: uint64(v.Proposal.Duration()),
DealID: v.DealID,
CreationTime: v.CreationTime.Time(),
Verified: v.Proposal.VerifiedDeal,
}
}
@@ -229,7 +222,6 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo,
Duration: uint64(v.Proposal.Duration()),
DealID: v.DealID,
CreationTime: v.CreationTime.Time(),
Verified: v.Proposal.VerifiedDeal,
}, nil
}
@@ -855,7 +847,6 @@ func newDealInfo(v storagemarket.ClientDeal) api.DealInfo {
Duration: uint64(v.Proposal.Duration()),
DealID: v.DealID,
CreationTime: v.CreationTime.Time(),
Verified: v.Proposal.VerifiedDeal,
}
}
-6
View File
@@ -121,12 +121,6 @@ func (a *CommonAPI) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo,
func (a *CommonAPI) NetAutoNatStatus(ctx context.Context) (i api.NatInfo, err error) {
autonat := a.RawHost.(*basichost.BasicHost).AutoNat
if autonat == nil {
return api.NatInfo{
Reachability: network.ReachabilityUnknown,
}, nil
}
var maddr string
if autonat.Status() == network.ReachabilityPublic {
pa, err := autonat.PublicAddr()
-9
View File
@@ -1,8 +1,6 @@
package impl
import (
"context"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/lotus/api"
@@ -11,7 +9,6 @@ import (
"github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/node/impl/market"
"github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
var log = logging.Logger("node")
@@ -29,12 +26,6 @@ type FullNodeAPI struct {
full.WalletAPI
full.SyncAPI
full.BeaconAPI
DS dtypes.MetadataDS
}
func (n *FullNodeAPI) CreateBackup(ctx context.Context, fpath string) error {
return backup(n.DS, fpath)
}
var _ api.FullNode = &FullNodeAPI{}
+13 -19
View File
@@ -509,11 +509,15 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipo
r, w := io.Pipe()
out := make(chan []byte)
go func() {
bw := bufio.NewWriterSize(w, 1<<20)
defer w.Close() //nolint:errcheck // it is a pipe
err := a.Chain.Export(ctx, ts, nroots, skipoldmsgs, bw)
bw.Flush() //nolint:errcheck // it is a write to a pipe
w.CloseWithError(err) //nolint:errcheck // it is a pipe
bw := bufio.NewWriterSize(w, 1<<20)
defer bw.Flush() //nolint:errcheck // it is a write to a pipe
if err := a.Chain.Export(ctx, ts, nroots, skipoldmsgs, bw); err != nil {
log.Errorf("chain export call failed: %s", err)
return
}
}()
go func() {
@@ -525,23 +529,13 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipo
log.Errorf("chain export pipe read failed: %s", err)
return
}
if n > 0 {
select {
case out <- buf[:n]:
case <-ctx.Done():
log.Warnf("export writer failed: %s", ctx.Err())
return
}
select {
case out <- buf[:n]:
case <-ctx.Done():
log.Warnf("export writer failed: %s", ctx.Err())
return
}
if err == io.EOF {
// send empty slice to indicate correct eof
select {
case out <- []byte{}:
case <-ctx.Done():
log.Warnf("export writer failed: %s", ctx.Err())
return
}
return
}
}
+11 -11
View File
@@ -110,10 +110,6 @@ func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (ci
return a.Mpool.Push(smsg)
}
func (a *MpoolAPI) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) {
return a.Mpool.PushUntrusted(smsg)
}
func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
cp := *msg
msg = &cp
@@ -160,13 +156,17 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value)
}
// Sign and push the message
return a.MessageSigner.SignMessage(ctx, msg, func(smsg *types.SignedMessage) error {
if _, err := a.Mpool.Push(smsg); err != nil {
return xerrors.Errorf("mpool push: failed to push message: %w", err)
}
return nil
})
smsg, err := a.MessageSigner.SignMessage(ctx, msg)
if err != nil {
return nil, xerrors.Errorf("mpool push: failed to sign message: %w", err)
}
_, err = a.Mpool.Push(smsg)
if err != nil {
return nil, xerrors.Errorf("mpool push: failed to push message: %w", err)
}
return smsg, err
}
func (a *MpoolAPI) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) {
-45
View File
@@ -1019,37 +1019,6 @@ func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address
return types.BigAdd(abal, vested), nil
}
// StateVerifiedClientStatus returns the data cap for the given address.
// Returns zero if there is no entry in the data cap table for the
// address.
func (a *StateAPI) StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) {
act, err := a.StateGetActor(ctx, verifreg.Address, tsk)
if err != nil {
return nil, err
}
aid, err := a.StateLookupID(ctx, addr, tsk)
if err != nil {
log.Warnf("lookup failure %v", err)
return nil, err
}
vrs, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load verified registry state: %w", err)
}
verified, dcap, err := vrs.VerifierDataCap(aid)
if err != nil {
return nil, xerrors.Errorf("looking up verifier: %w", err)
}
if !verified {
return nil, nil
}
return &dcap, nil
}
// StateVerifiedClientStatus returns the data cap for the given address.
// Returns zero if there is no entry in the data cap table for the
// address.
@@ -1081,20 +1050,6 @@ func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.A
return &dcap, nil
}
func (a *StateAPI) StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) {
vact, err := a.StateGetActor(ctx, verifreg.Address, tsk)
if err != nil {
return address.Undef, err
}
vst, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), vact)
if err != nil {
return address.Undef, err
}
return vst.RootKey()
}
var dealProviderCollateralNum = types.NewInt(110)
var dealProviderCollateralDen = types.NewInt(100)
+2 -8
View File
@@ -8,18 +8,18 @@ import (
"strconv"
"time"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/big"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/host"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/piecestore"
retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket"
storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
@@ -56,8 +56,6 @@ type StorageMinerAPI struct {
DataTransfer dtypes.ProviderDataTransfer
Host host.Host
DS dtypes.MetadataDS
ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc
SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc
ConsiderOnlineRetrievalDealsConfigFunc dtypes.ConsiderOnlineRetrievalDealsConfigFunc
@@ -518,8 +516,4 @@ func (sm *StorageMinerAPI) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.
return &ci, nil
}
func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error {
return backup(sm.DS, fpath)
}
var _ api.StorageMiner = &StorageMinerAPI{}
+2 -2
View File
@@ -223,8 +223,8 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
},
AppSpecificWeight: 1,
// This sets the IP colocation threshold to 5 peers before we apply penalties
IPColocationFactorThreshold: 5,
// This sets the IP colocation threshold to 1 peer per
IPColocationFactorThreshold: 1,
IPColocationFactorWeight: -100,
// TODO we want to whitelist IPv6 /64s that belong to datacenters etc
// IPColocationFactorWhitelist: map[string]struct{}{},
+1 -7
View File
@@ -6,7 +6,6 @@ import (
"go.uber.org/fx"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/backupds"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo"
)
@@ -28,10 +27,5 @@ func KeyStore(lr repo.LockedRepo) (types.KeyStore, error) {
}
func Datastore(r repo.LockedRepo) (dtypes.MetadataDS, error) {
mds, err := r.Datastore("/metadata")
if err != nil {
return nil, err
}
return backupds.Wrap(mds), nil
return r.Datastore("/metadata")
}
-8
View File
@@ -44,7 +44,6 @@ import (
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-storedcounter"
"github.com/filecoin-project/specs-actors/actors/builtin"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
@@ -469,13 +468,6 @@ func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnline
return false, fmt.Sprintf("cannot seal a sector before %s", deal.Proposal.StartEpoch), nil
}
// Reject if it's more than 7 days in the future
// TODO: read from cfg
maxStartEpoch := ht + abi.ChainEpoch(7*builtin.EpochsInDay)
if deal.Proposal.StartEpoch > maxStartEpoch {
return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil
}
if user != nil {
return user(ctx, deal)
}
-12
View File
@@ -226,23 +226,11 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) {
}, nil
}
// Like Lock, except datastores will work in read-only mode
func (fsr *FsRepo) LockRO(repoType RepoType) (LockedRepo, error) {
lr, err := fsr.Lock(repoType)
if err != nil {
return nil, err
}
lr.(*fsLockedRepo).readonly = true
return lr, nil
}
type fsLockedRepo struct {
path string
configPath string
repoType RepoType
closer io.Closer
readonly bool
ds map[string]datastore.Batching
dsErr error
+7 -11
View File
@@ -14,7 +14,7 @@ import (
ldbopts "github.com/syndtr/goleveldb/leveldb/opt"
)
type dsCtor func(path string, readonly bool) (datastore.Batching, error)
type dsCtor func(path string) (datastore.Batching, error)
var fsDatastores = map[string]dsCtor{
"chain": chainBadgerDs,
@@ -26,10 +26,9 @@ var fsDatastores = map[string]dsCtor{
"client": badgerDs, // client specific
}
func chainBadgerDs(path string, readonly bool) (datastore.Batching, error) {
func chainBadgerDs(path string) (datastore.Batching, error) {
opts := badger.DefaultOptions
opts.GcInterval = 0 // disable GC for chain datastore
opts.ReadOnly = readonly
opts.Options = dgbadger.DefaultOptions("").WithTruncate(true).
WithValueThreshold(1 << 10)
@@ -37,26 +36,23 @@ func chainBadgerDs(path string, readonly bool) (datastore.Batching, error) {
return badger.NewDatastore(path, &opts)
}
func badgerDs(path string, readonly bool) (datastore.Batching, error) {
func badgerDs(path string) (datastore.Batching, error) {
opts := badger.DefaultOptions
opts.ReadOnly = readonly
opts.Options = dgbadger.DefaultOptions("").WithTruncate(true).
WithValueThreshold(1 << 10)
return badger.NewDatastore(path, &opts)
}
func levelDs(path string, readonly bool) (datastore.Batching, error) {
func levelDs(path string) (datastore.Batching, error) {
return levelds.NewDatastore(path, &levelds.Options{
Compression: ldbopts.NoCompression,
NoSync: false,
Strict: ldbopts.StrictAll,
ReadOnly: readonly,
})
}
func (fsr *fsLockedRepo) openDatastores(readonly bool) (map[string]datastore.Batching, error) {
func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error) {
if err := os.MkdirAll(fsr.join(fsDatastore), 0755); err != nil {
return nil, xerrors.Errorf("mkdir %s: %w", fsr.join(fsDatastore), err)
}
@@ -67,7 +63,7 @@ func (fsr *fsLockedRepo) openDatastores(readonly bool) (map[string]datastore.Bat
prefix := datastore.NewKey(p)
// TODO: optimization: don't init datastores we don't need
ds, err := ctor(fsr.join(filepath.Join(fsDatastore, p)), readonly)
ds, err := ctor(fsr.join(filepath.Join(fsDatastore, p)))
if err != nil {
return nil, xerrors.Errorf("opening datastore %s: %w", prefix, err)
}
@@ -82,7 +78,7 @@ func (fsr *fsLockedRepo) openDatastores(readonly bool) (map[string]datastore.Bat
func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) {
fsr.dsOnce.Do(func() {
fsr.ds, fsr.dsErr = fsr.openDatastores(fsr.readonly)
fsr.ds, fsr.dsErr = fsr.openDatastores()
})
if fsr.dsErr != nil {

Some files were not shown because too many files have changed in this diff Show More