cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #379 from filecoin-project/feat/variable-sector-sizes

Browse Source 
Allow miners to create differently sized sectors
This commit is contained in:
Łukasz Magiera 2019-10-25 17:37:54 +02:00 committed by GitHub
commit bbe5afeee4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 145 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/api.go
View File

@ -129,6 +129,7 @@ type FullNode interface {
StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error)
StateMinerPeerID(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error)
StateMinerProvingPeriodEnd(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error)
StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error)
StatePledgeCollateral(context.Context, *types.TipSet) (types.BigInt, error)
StateWaitMsg(context.Context, cid.Cid) (*MsgWait, error)
StateListMiners(context.Context, *types.TipSet) ([]address.Address, error)

6
api/struct.go
View File

@ -92,6 +92,7 @@ type FullNodeStruct struct {
StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"`
StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"`
StateMinerProvingPeriodEnd func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"`
StateMinerSectorSize func(context.Context, address.Address, *types.TipSet) (uint64, error) `perm:"read"`
StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"`
StateReplay func(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error) `perm:"read"`
StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"`
@ -357,10 +358,15 @@ func (c *FullNodeStruct) StateMinerWorker(ctx context.Context, m address.Address
func (c *FullNodeStruct) StateMinerPeerID(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) {
return c.Internal.StateMinerPeerID(ctx, m, ts)
}
func (c *FullNodeStruct) StateMinerProvingPeriodEnd(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) {
return c.Internal.StateMinerProvingPeriodEnd(ctx, actor, ts)
}
func (c *FullNodeStruct) StateMinerSectorSize(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) {
return c.Internal.StateMinerSectorSize(ctx, actor, ts)
}
func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) {
return c.Internal.StateCall(ctx, msg, ts)
}

2
build/paramfetch.go
View File

@ -39,7 +39,7 @@ func GetParams(storage bool) error {
}
for name, info := range params {
if info.SectorSize != SectorSize {
if !SupportedSectorSize(info.SectorSize) {
continue
}
if !storage && strings.HasSuffix(name, ".params") {

19
build/params.go
View File

@ -1,6 +1,8 @@
package build
import "math/big"
import (
"math/big"
)
// Core network constants
@ -10,7 +12,20 @@ import "math/big"
const UnixfsChunkSize uint64 = 1 << 20
const UnixfsLinksPerLevel = 1024
const SectorSize = 16 << 20
var SectorSizes = []uint64{
16 << 20,
256 << 20,
1 << 30,
}
func SupportedSectorSize(ssize uint64) bool {
for _, ss := range SectorSizes {
if ssize == ss {
return true
}
}
return false
}
// /////
// Payments

16
chain/actors/actor_miner.go
View File

@ -87,13 +87,13 @@ type MinerInfo struct {
PeerID peer.ID
// Amount of space in each sector committed to the network by this miner.
SectorSize types.BigInt
SectorSize uint64
}
type StorageMinerConstructorParams struct {
Owner address.Address
Worker address.Address
SectorSize types.BigInt
SectorSize uint64
PeerID peer.ID
}
@ -241,7 +241,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex
}
// Power of the miner after adding this sector
futurePower := types.BigAdd(self.Power, mi.SectorSize)
futurePower := types.BigAdd(self.Power, types.NewInt(mi.SectorSize))
collateralRequired := CollateralForPower(futurePower)
// TODO: grab from market?
@ -399,7 +399,7 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext,
faults := self.CurrentFaultSet.All()
if ok, lerr := sectorbuilder.VerifyPost(mi.SectorSize.Uint64(),
if ok, lerr := sectorbuilder.VerifyPost(mi.SectorSize,
sectorbuilder.NewSortedSectorInfo(sectorInfos), seed, params.Proof,
faults); !ok || lerr != nil {
if lerr != nil {
@ -432,7 +432,7 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext,
oldPower := self.Power
self.Power = types.BigMul(types.NewInt(pss.Count-uint64(len(faults))),
mi.SectorSize)
types.NewInt(mi.SectorSize))
enc, err := SerializeParams(&UpdateStorageParams{Delta: types.BigSub(self.Power, oldPower)})
if err != nil {
@ -517,8 +517,8 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID
return true, comms[0], comms[1], nil
}
func ValidatePoRep(maddr address.Address, ssize types.BigInt, params *OnChainSealVerifyInfo) (bool, ActorError) {
ok, err := sectorbuilder.VerifySeal(ssize.Uint64(), params.CommR, params.CommD, params.CommRStar, maddr, params.SectorNumber, params.Proof)
func ValidatePoRep(maddr address.Address, ssize uint64, params *OnChainSealVerifyInfo) (bool, ActorError) {
ok, err := sectorbuilder.VerifySeal(ssize, params.CommR, params.CommD, params.CommRStar, maddr, params.SectorNumber, params.Proof)
if err != nil {
return false, aerrors.Absorb(err, 25, "verify seal failed")
}
@ -632,7 +632,7 @@ func (sma StorageMinerActor) GetSectorSize(act *types.Actor, vmctx types.VMConte
return nil, err
}
return mi.SectorSize.Bytes(), nil
return types.NewInt(mi.SectorSize).Bytes(), nil
}
type PaymentVerifyParams struct {

11
chain/actors/actor_storagepower.go
View File

@ -53,12 +53,12 @@ type StoragePowerState struct {
type CreateStorageMinerParams struct {
Owner address.Address
Worker address.Address
SectorSize types.BigInt
SectorSize uint64
PeerID peer.ID
}
func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VMContext, params *CreateStorageMinerParams) ([]byte, ActorError) {
if !SupportedSectorSize(params.SectorSize) {
if !build.SupportedSectorSize(params.SectorSize) {
return nil, aerrors.New(1, "Unsupported sector size")
}
@ -116,13 +116,6 @@ func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VM
return naddr.Bytes(), nil
}
func SupportedSectorSize(ssize types.BigInt) bool {
if ssize.Uint64() == build.SectorSize {
return true
}
return false
}
type ArbitrateConsensusFaultParams struct {
Block1 *types.BlockHeader
Block2 *types.BlockHeader

2
chain/actors/actor_storagepower_test.go
View File

@ -42,7 +42,7 @@ func TestStorageMarketCreateAndSlashMiner(t *testing.T) {
&CreateStorageMinerParams{
Owner: ownerAddr,
Worker: workerAddr,
SectorSize: types.NewInt(build.SectorSize),
SectorSize: build.SectorSizes[0],
PeerID: "fakepeerid",
})
ApplyOK(t, ret)

2
chain/actors/actors_test.go
View File

@ -118,7 +118,7 @@ func TestStorageMarketActorCreateMiner(t *testing.T) {
params := &StorageMinerConstructorParams{
Owner: maddr,
Worker: maddr,
SectorSize: types.NewInt(build.SectorSize),
SectorSize: build.SectorSizes[0],
PeerID: "fakepeerid",
}
var err error

57
chain/actors/cbor_gen.go
View File

@ -421,8 +421,8 @@ func (t *StorageMinerConstructorParams) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.SectorSize (types.BigInt)
if err := t.SectorSize.MarshalCBOR(w); err != nil {
// t.t.SectorSize (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil {
return err
}
@ -469,15 +469,16 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error {
}
}
// t.t.SectorSize (types.BigInt)
{
if err := t.SectorSize.UnmarshalCBOR(br); err != nil {
return err
}
// t.t.SectorSize (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorSize = extra
// t.t.PeerID (peer.ID)
{
@ -702,8 +703,8 @@ func (t *MinerInfo) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.SectorSize (types.BigInt)
if err := t.SectorSize.MarshalCBOR(w); err != nil {
// t.t.SectorSize (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil {
return err
}
return nil
@ -752,15 +753,16 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error {
t.PeerID = peer.ID(sval)
}
// t.t.SectorSize (types.BigInt)
{
if err := t.SectorSize.UnmarshalCBOR(br); err != nil {
return err
}
// t.t.SectorSize (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorSize = extra
return nil
}
@ -2436,8 +2438,8 @@ func (t *CreateStorageMinerParams) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.SectorSize (types.BigInt)
if err := t.SectorSize.MarshalCBOR(w); err != nil {
// t.t.SectorSize (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil {
return err
}
@ -2484,15 +2486,16 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error {
}
}
// t.t.SectorSize (types.BigInt)
{
if err := t.SectorSize.UnmarshalCBOR(br); err != nil {
return err
}
// t.t.SectorSize (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorSize = extra
// t.t.PeerID (peer.ID)
{

12
chain/deals/client_states.go
View File

@ -2,11 +2,12 @@ package deals
import (
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/stmgr"
"golang.org/x/xerrors"
)
type clientHandlerFunc func(ctx context.Context, deal ClientDeal) error
@ -108,7 +109,12 @@ func (c *Client) staged(ctx context.Context, deal ClientDeal) error {
log.Info("DEAL SEALED!")
// TODO: want?
/*ok, err := sectorbuilder.VerifyPieceInclusionProof(build.SectorSize, deal.Proposal.PieceSize, deal.Proposal.CommP, resp.CommD, resp.PieceInclusionProof.ProofElements)
/*ssize, err := stmgr.GetMinerSectorSize(ctx, c.sm, nil, deal.Proposal.MinerAddress)
if err != nil {
return xerrors.Errorf("failed to get miner sector size: %w", err)
}
ok, err := sectorbuilder.VerifyPieceInclusionProof(ssize, deal.Proposal.Size, deal.Proposal.CommP, resp.CommD, resp.PieceInclusionProof.ProofElements)
if err != nil {
return xerrors.Errorf("verifying piece inclusion proof in staged deal %s: %w", deal.ProposalCid, err)
}

14
chain/gen/gen.go
View File

@ -208,12 +208,12 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add
worker, err := stmgr.GetMinerWorkerRaw(ctx, cg.sm, st, m)
if err != nil {
return nil, nil, err
return nil, nil, xerrors.Errorf("get miner worker: %w", err)
}
vrfout, err := ComputeVRF(ctx, cg.w.Sign, worker, lastTicket.VRFProof)
if err != nil {
return nil, nil, err
return nil, nil, xerrors.Errorf("compute VRF: %w", err)
}
tick := &types.Ticket{
@ -252,7 +252,7 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad
msgs, err := cg.getRandomMessages()
if err != nil {
return nil, err
return nil, xerrors.Errorf("get random messages: %w", err)
}
for len(blks) == 0 {
@ -279,6 +279,12 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad
}
fts := store.NewFullTipSet(blks)
fmt.Println("Made a block: ", fts.TipSet().Cids())
if len(fts.TipSet().Cids()) > 1 {
for _, b := range blks {
fmt.Printf("block %s: %#v\n", b.Cid(), b.Header)
}
}
return &MinedTipSet{
TipSet: fts,
@ -395,7 +401,7 @@ func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*ty
func IsRoundWinner(ctx context.Context, ts *types.TipSet, ticks []*types.Ticket, miner address.Address, a MiningCheckAPI) (bool, types.ElectionProof, error) {
r, err := a.ChainGetRandomness(ctx, ts, ticks, build.RandomnessLookback)
if err != nil {
return false, nil, err
return false, nil, xerrors.Errorf("chain get randomness: %w", err)
}
mworker, err := a.StateMinerWorker(ctx, miner, ts)

2
chain/gen/utils.go
View File

@ -239,7 +239,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
params := mustEnc(&actors.CreateStorageMinerParams{
Owner: owner,
Worker: worker,
SectorSize: types.NewInt(build.SectorSize),
SectorSize: build.SectorSizes[0],
PeerID: pid,
})

18
chain/stmgr/utils.go
View File

@ -2,6 +2,7 @@ package stmgr
import (
"context"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
@ -9,6 +10,7 @@ import (
amt "github.com/filecoin-project/go-amt-ipld"
cid "github.com/ipfs/go-cid"
hamt "github.com/ipfs/go-hamt-ipld"
blockstore "github.com/ipfs/go-ipfs-blockstore"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/libp2p/go-libp2p-core/peer"
@ -175,6 +177,22 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet,
return LoadSectorsFromSet(ctx, sm.ChainStore().Blockstore(), mas.Sectors)
}
func GetMinerSectorSize(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (uint64, error) {
var mas actors.StorageMinerActorState
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return 0, xerrors.Errorf("failed to load miner actor state: %w", err)
}
cst := hamt.CSTFromBstore(sm.cs.Blockstore())
var minfo actors.MinerInfo
if err := cst.Get(ctx, mas.Info, &minfo); err != nil {
return 0, xerrors.Errorf("failed to read miner info: %w", err)
}
return minfo.SectorSize, nil
}
func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid) ([]*api.SectorInfo, error) {
blks := amt.WrapBlockstore(bs)
a, err := amt.LoadAMT(blks, ssc)

2
cli/createminer.go
View File

@ -53,7 +53,7 @@ var createMinerCmd = &cli.Command{
createMinerArgs := actors.CreateStorageMinerParams{
Worker: worker,
Owner: owner,
SectorSize: types.NewInt(ssize),
SectorSize: ssize,
PeerID: pid,
}

9
cmd/lotus-storage-miner/init.go
View File

@ -49,6 +49,11 @@ var initCmd = &cli.Command{
Aliases: []string{"o"},
Usage: "owner key to use",
},
&cli.Uint64Flag{
Name: "sector-size",
Usage: "specify sector size to use",
Value: build.SectorSizes[0],
},
},
Action: func(cctx *cli.Context) error {
log.Info("Initializing lotus storage miner")
@ -276,6 +281,8 @@ func createStorageMiner(ctx context.Context, api api.FullNode, peerid peer.ID, c
return address.Undef, err
}
ssize := cctx.Uint64("sector-size")
worker := owner
if cctx.String("worker") != "" {
worker, err = address.NewFromString(cctx.String("worker"))
@ -295,7 +302,7 @@ func createStorageMiner(ctx context.Context, api api.FullNode, peerid peer.ID, c
params, err := actors.SerializeParams(&actors.CreateStorageMinerParams{
Owner: owner,
Worker: worker,
SectorSize: types.NewInt(build.SectorSize),
SectorSize: ssize,
PeerID: peerid,
})
if err != nil {

4
node/impl/full/state.go
View File

@ -69,6 +69,10 @@ func (a *StateAPI) StateMinerProvingPeriodEnd(ctx context.Context, actor address
return stmgr.GetMinerProvingPeriodEnd(ctx, a.StateManager, ts, actor)
}
func (a *StateAPI) StateMinerSectorSize(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) {
return stmgr.GetMinerSectorSize(ctx, a.StateManager, ts, actor)
}
func (a *StateAPI) StatePledgeCollateral(ctx context.Context, ts *types.TipSet) (types.BigInt, error) {
param, err := actors.SerializeParams(&actors.PledgeCollateralParams{Size: types.NewInt(0)})
if err != nil {

9
node/impl/storminer.go
View File

@ -7,12 +7,13 @@ import (
"math/rand"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sector"
"github.com/filecoin-project/lotus/storage/sectorblocks"
"golang.org/x/xerrors"
)
type StorageMinerAPI struct {
@ -31,7 +32,11 @@ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error
}
func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) {
size := sectorbuilder.UserBytesForSectorSize(build.SectorSize)
ssize, err := sm.Miner.SectorSize(ctx)
if err != nil {
return 0, xerrors.Errorf("failed to get miner sector size: %w", err)
}
size := sectorbuilder.UserBytesForSectorSize(ssize)
// TODO: create a deal
name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000))

12
node/modules/storageminer.go
View File

@ -16,7 +16,6 @@ import (
"go.uber.org/fx"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
@ -38,13 +37,18 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
return address.NewFromBytes(maddrb)
}
func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS) (*sectorbuilder.SectorBuilderConfig, error) {
return func(ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilderConfig, error) {
func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) {
return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) {
minerAddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
}
ssize, err := api.StateMinerSectorSize(context.TODO(), minerAddr, nil)
if err != nil {
return nil, err
}
sp, err := homedir.Expand(storagePath)
if err != nil {
return nil, err
@ -56,7 +60,7 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS) (*sectorbui
sb := &sectorbuilder.SectorBuilderConfig{
Miner: minerAddr,
SectorSize: build.SectorSize,
SectorSize: ssize,
MetadataDir: metadata,
SealedDir: sealed,
StagedDir: staging,

14
storage/miner.go
View File

@ -12,7 +12,6 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/events"
@ -55,6 +54,7 @@ type storageMinerApi interface {
StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error)
StateMinerProvingPeriodEnd(context.Context, address.Address, *types.TipSet) (uint64, error)
StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error)
StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error)
StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error)
MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error)
@ -122,7 +122,12 @@ func (m *Miner) handlePostingSealedSectors(ctx context.Context) {
func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSealingStatus) error {
log.Info("committing sector")
ok, err := sectorbuilder.VerifySeal(build.SectorSize, sinfo.CommR[:], sinfo.CommD[:], sinfo.CommRStar[:], m.maddr, sinfo.SectorID, sinfo.Proof)
ssize, err := m.SectorSize(ctx)
if err != nil {
return xerrors.Errorf("failed to check out own sector size: %w", err)
}
ok, err := sectorbuilder.VerifySeal(ssize, sinfo.CommR[:], sinfo.CommD[:], sinfo.CommRStar[:], m.maddr, sinfo.SectorID, sinfo.Proof)
if err != nil {
log.Error("failed to verify seal we just created: ", err)
}
@ -200,3 +205,8 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error {
log.Infof("starting up miner %s, worker addr %s", m.maddr, m.worker)
return nil
}
func (m *Miner) SectorSize(ctx context.Context) (uint64, error) {
// TODO: cache this
return m.api.StateMinerSectorSize(ctx, m.maddr, nil)
}