cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Implement committed capacity sectors

Browse Source
This commit is contained in:
Łukasz Magiera 2020-02-23 01:47:47 +01:00
parent 1381baf088
commit 226685dc1c
21 changed files with 400 additions and 343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
api/cbor_gen.go
View File

@ -244,6 +244,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil {
return err
}
return nil
}
@ -280,36 +281,48 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
// t.SectorID (abi.SectorNumber) (uint64)
case "SectorID":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = abi.SectorNumber(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = abi.SectorNumber(extra)
// t.Offset (uint64) (uint64)
case "Offset":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Offset = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Offset = uint64(extra)
// t.Size (abi.UnpaddedPieceSize) (uint64)
case "Size":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Size = abi.UnpaddedPieceSize(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Size = abi.UnpaddedPieceSize(extra)
default:
return fmt.Errorf("unknown struct field %d: '%s'", i, name)

58
chain/blocksync/cbor_gen.go
View File

@ -38,14 +38,17 @@ func (t *BlockSyncRequest) MarshalCBOR(w io.Writer) error {
}
// t.RequestLength (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.RequestLength))); err != nil {
return err
}
// t.Options (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Options))); err != nil {
return err
}
return nil
}
@ -92,24 +95,32 @@ func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error {
// t.RequestLength (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.RequestLength = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.RequestLength = uint64(extra)
// t.Options (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Options = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Options = uint64(extra)
return nil
}
@ -137,6 +148,7 @@ func (t *BlockSyncResponse) MarshalCBOR(w io.Writer) error {
}
// t.Status (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil {
return err
}
@ -199,14 +211,18 @@ func (t *BlockSyncResponse) UnmarshalCBOR(r io.Reader) error {
// t.Status (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Status = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Status = uint64(extra)
// t.Message (string) (string)
{

2
chain/sync.go
View File

@ -512,7 +512,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
if h.Timestamp < baseTs.MinTimestamp()+(build.BlockDelay*uint64(h.Height-baseTs.Height())) {
log.Warn("timestamp funtimes: ", h.Timestamp, baseTs.MinTimestamp(), h.Height, baseTs.Height())
diff := (baseTs.MinTimestamp()+(build.BlockDelay*uint64(h.Height-baseTs.Height()))) - h.Timestamp
diff := (baseTs.MinTimestamp() + (build.BlockDelay * uint64(h.Height-baseTs.Height()))) - h.Timestamp
return xerrors.Errorf("block was generated too soon (h.ts:%d < base.mints:%d + BLOCK_DELAY:%d * deltaH:%d; diff %d)", h.Timestamp, baseTs.MinTimestamp(), build.BlockDelay, h.Height-baseTs.Height(), diff)
}

135
chain/types/cbor_gen.go
View File

@ -94,6 +94,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
}
// t.Timestamp (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil {
return err
}
@ -104,9 +105,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
}
// t.ForkSignaling (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ForkSignaling))); err != nil {
return err
}
return nil
}
@ -271,14 +274,18 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
}
// t.Timestamp (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Timestamp = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Timestamp = uint64(extra)
// t.BlockSig (crypto.Signature) (struct)
{
@ -302,14 +309,18 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
}
// t.ForkSignaling (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ForkSignaling = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ForkSignaling = uint64(extra)
return nil
}
@ -521,14 +532,17 @@ func (t *EPostTicket) MarshalCBOR(w io.Writer) error {
}
// t.SectorID (abi.SectorNumber) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
return err
}
// t.ChallengeIndex (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ChallengeIndex))); err != nil {
return err
}
return nil
}
@ -566,24 +580,32 @@ func (t *EPostTicket) UnmarshalCBOR(r io.Reader) error {
}
// t.SectorID (abi.SectorNumber) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = abi.SectorNumber(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = abi.SectorNumber(extra)
// t.ChallengeIndex (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ChallengeIndex = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ChallengeIndex = uint64(extra)
return nil
}
@ -607,6 +629,7 @@ func (t *Message) MarshalCBOR(w io.Writer) error {
}
// t.Nonce (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil {
return err
}
@ -627,6 +650,7 @@ func (t *Message) MarshalCBOR(w io.Writer) error {
}
// t.Method (abi.MethodNum) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil {
return err
}
@ -680,14 +704,18 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error {
}
// t.Nonce (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Nonce = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Nonce = uint64(extra)
// t.Value (big.Int) (struct)
{
@ -717,14 +745,18 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error {
}
// t.Method (abi.MethodNum) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Method = abi.MethodNum(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Method = abi.MethodNum(extra)
// t.Params ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
@ -890,6 +922,7 @@ func (t *Actor) MarshalCBOR(w io.Writer) error {
}
// t.Nonce (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil {
return err
}
@ -942,14 +975,18 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error {
}
// t.Nonce (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Nonce = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Nonce = uint64(extra)
// t.Balance (big.Int) (struct)
{

31
chain/vm/syscalls.go
View File

@ -3,13 +3,16 @@ package vm
import (
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-address"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/lib/zerocomm"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/runtime"
"github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
"math/bits"
)
func init() {
@ -29,11 +32,30 @@ type syscallShim struct {
func (ss *syscallShim) ComputeUnsealedSectorCID(ssize abi.SectorSize, pieces []abi.PieceInfo) (cid.Cid, error) {
// TODO: does this pull in unwanted dependencies?
var ffipieces []ffi.PublicPieceInfo
var sum abi.PaddedPieceSize
for _, p := range pieces {
ffipieces = append(ffipieces, ffi.PublicPieceInfo{
Size: p.Size.Unpadded(),
CommP: cidToCommD(p.PieceCID),
})
sum += p.Size
}
{
// pad remaining space with 0 CommPs
toFill := uint64(abi.PaddedPieceSize(ssize) - sum)
n := bits.OnesCount64(toFill)
for i := 0; i < n; i++ {
next := bits.TrailingZeros64(toFill)
psize := uint64(1) << next
toFill ^= psize
unpadded := abi.PaddedPieceSize(psize).Unpadded()
ffipieces = append(ffipieces, ffi.PublicPieceInfo{
Size: unpadded,
CommP: zerocomm.ForSize(unpadded),
})
}
}
commd, err := sectorbuilder.GenerateDataCommitment(ssize, ffipieces)
@ -42,14 +64,7 @@ func (ss *syscallShim) ComputeUnsealedSectorCID(ssize abi.SectorSize, pieces []a
return cid.Undef, err
}
// TODO: pulling these numbers kinda out of an unmerged PR
hb, err := mh.Encode(commd[:], 0xf104)
if err != nil {
log.Errorf("mh encode failed: %s", err)
return cid.Undef, err
}
return cid.NewCidV1(0xf101, mh.Multihash(hb)), nil
return commcid.DataCommitmentV1ToCID(commd[:]), nil
}
func (ss *syscallShim) HashBlake2b(data []byte) [32]byte {

2
cli/state.go
View File

@ -274,10 +274,8 @@ var stateReplaySetCmd = &cli.Command{
return err
}
}
res, err := api.StateReplay(ctx, ts, mcid)
if err != nil {
return xerrors.Errorf("replay call failed: %w", err)

10
cmd/lotus-seed/genesis.go
View File

@ -39,15 +39,15 @@ var genesisNewCmd = &cli.Command{
}
out := genesis.Template{
Accounts: []genesis.Actor{},
Miners: []genesis.Miner{},
Accounts: []genesis.Actor{},
Miners: []genesis.Miner{},
NetworkName: cctx.String("network-name"),
}
if out.NetworkName == "" {
out.NetworkName = "localnet-" + uuid.New().String()
}
genb, err := json.MarshalIndent(&out, ""," ")
genb, err := json.MarshalIndent(&out, "", " ")
if err != nil {
return err
}
@ -68,7 +68,7 @@ var genesisNewCmd = &cli.Command{
var genesisAddMinerCmd = &cli.Command{
Name: "add-miner",
Description: "add genesis miner",
Flags: []cli.Flag{},
Flags: []cli.Flag{},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 2 {
return xerrors.New("seed genesis add-miner [genesis.json] [preseal.json]")
@ -128,7 +128,7 @@ var genesisAddMinerCmd = &cli.Command{
})
}
genb, err = json.MarshalIndent(&template, ""," ")
genb, err = json.MarshalIndent(&template, "", " ")
if err != nil {
return err
}

2
cmd/lotus-storage-miner/init.go
View File

@ -250,7 +250,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin
SectorID: sector.SectorID,
Pieces: []sealing.Piece{
{
DealID: dealID,
DealID: &dealID,
Size: abi.PaddedPieceSize(meta.SectorSize).Unpadded(),
CommP: sector.CommD[:],
},

2
go.mod
View File

@ -88,7 +88,7 @@ require (
github.com/prometheus/common v0.4.0
github.com/stretchr/testify v1.4.0
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
github.com/whyrusleeping/cbor-gen v0.0.0-20200213013405-80352c7ae952
github.com/whyrusleeping/cbor-gen v0.0.0-20200222160900-51052a1e8191
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
go.opencensus.io v0.22.2

2
go.sum
View File

@ -762,6 +762,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66 h1:LolR9FiE
github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
github.com/whyrusleeping/cbor-gen v0.0.0-20200213013405-80352c7ae952 h1:YGoOfwBeeb6A8t3eSe2nmPtMsyYPa6WzBmTbSPLRNK8=
github.com/whyrusleeping/cbor-gen v0.0.0-20200213013405-80352c7ae952/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
github.com/whyrusleeping/cbor-gen v0.0.0-20200222160900-51052a1e8191 h1:TeuxLwKwQy612jEhfVhGJTqLsM2EwMi1eJE052ug+NY=
github.com/whyrusleeping/cbor-gen v0.0.0-20200222160900-51052a1e8191/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=

56
lib/statemachine/cbor_gen.go
View File

@ -52,6 +52,7 @@ func (t *TestState) MarshalCBOR(w io.Writer) error {
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.B))); err != nil {
return err
}
return nil
}
@ -88,25 +89,33 @@ func (t *TestState) UnmarshalCBOR(r io.Reader) error {
// t.A (uint64) (uint64)
case "A":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.A = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.A = uint64(extra)
// t.B (uint64) (uint64)
case "B":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.B = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.B = uint64(extra)
default:
return fmt.Errorf("unknown struct field %d: '%s'", i, name)
@ -162,6 +171,7 @@ func (t *TestEvent) MarshalCBOR(w io.Writer) error {
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Val))); err != nil {
return err
}
return nil
}
@ -209,14 +219,18 @@ func (t *TestEvent) UnmarshalCBOR(r io.Reader) error {
// t.Val (uint64) (uint64)
case "Val":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Val = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Val = uint64(extra)
default:
return fmt.Errorf("unknown struct field %d: '%s'", i, name)

5
node/impl/storminer.go
View File

@ -185,7 +185,10 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb
deals := make([]abi.DealID, len(info.Pieces))
for i, piece := range info.Pieces {
deals[i] = piece.DealID
if piece.DealID == nil {
continue
}
deals[i] = *piece.DealID
}
log := make([]api.SectorLog, len(info.Log))

37
paychmgr/cbor_gen.go
View File

@ -254,6 +254,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextLane))); err != nil {
return err
}
return nil
}
@ -320,14 +321,18 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
// t.Direction (uint64) (uint64)
case "Direction":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Direction = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Direction = uint64(extra)
// t.Vouchers ([]*paychmgr.VoucherInfo) (slice)
case "Vouchers":
@ -359,14 +364,18 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
// t.NextLane (uint64) (uint64)
case "NextLane":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.NextLane = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.NextLane = uint64(extra)
default:
return fmt.Errorf("unknown struct field %d: '%s'", i, name)

130
storage/sealing/cbor_gen.go
View File

@ -310,8 +310,14 @@ func (t *Piece) MarshalCBOR(w io.Writer) error {
return err
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
if t.DealID == nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(*t.DealID))); err != nil {
return err
}
}
// t.Size (abi.UnpaddedPieceSize) (uint64)
@ -388,25 +394,45 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error {
// t.DealID (abi.DealID) (uint64)
case "DealID":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
typed := abi.DealID(extra)
t.DealID = &typed
}
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = abi.DealID(extra)
// t.Size (abi.UnpaddedPieceSize) (uint64)
case "Size":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Size = abi.UnpaddedPieceSize(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Size = abi.UnpaddedPieceSize(extra)
// t.CommP ([]uint8) (slice)
case "CommP":
@ -765,36 +791,48 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
// t.State (uint64) (uint64)
case "State":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
// t.SectorID (abi.SectorNumber) (uint64)
case "SectorID":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = abi.SectorNumber(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = abi.SectorNumber(extra)
// t.Nonce (uint64) (uint64)
case "Nonce":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Nonce = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Nonce = uint64(extra)
// t.Pieces ([]sealing.Piece) (slice)
case "Pieces":
@ -1147,14 +1185,18 @@ func (t *Log) UnmarshalCBOR(r io.Reader) error {
// t.Timestamp (uint64) (uint64)
case "Timestamp":
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Timestamp = uint64(extra)
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Timestamp = uint64(extra)
// t.Trace (string) (string)
case "Trace":

27
storage/sealing/checks.go
View File

@ -1,7 +1,12 @@
package sealing
import (
"bytes"
"context"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/lotus/lib/zerocomm"
"github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/multiformats/go-multihash"
@ -19,6 +24,7 @@ import (
type ErrApi struct{ error }
type ErrInvalidDeals struct{ error }
type ErrInvalidPiece struct{ error }
type ErrExpiredDeals struct{ error }
type ErrBadCommD struct{ error }
@ -36,7 +42,14 @@ func checkPieces(ctx context.Context, si SectorInfo, api sealingApi) error {
}
for i, piece := range si.Pieces {
deal, err := api.StateMarketStorageDeal(ctx, piece.DealID, nil)
if piece.DealID == nil {
exp := zerocomm.ForSize(piece.Size)
if string(piece.CommP) != string(exp[:]) {
return &ErrInvalidPiece{xerrors.Errorf("deal %d piece %d had non-zero CommP %+v", piece.DealID, i, piece.CommP)}
}
continue
}
deal, err := api.StateMarketStorageDeal(ctx, *piece.DealID, nil)
if err != nil {
return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", piece.DealID, i, err)}
}
@ -99,7 +112,17 @@ func checkSeal(ctx context.Context, maddr address.Address, si SectorInfo, api se
if r.ExitCode != 0 {
return &ErrBadCommD{xerrors.Errorf("receipt for ComputeDataCommitment had exit code %d", r.ExitCode)}
}
if string(r.Return) != string(si.CommD) {
var c cbg.CborCid
if err := c.UnmarshalCBOR(bytes.NewReader(r.Return)); err != nil {
return err
}
cd, err := commcid.CIDToDataCommitmentV1(cid.Cid(c))
if err != nil {
return err
}
if string(cd) != string(si.CommD) {
return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %x != %x", r.Return, si.CommD)}
}

2
storage/sealing/fsm.go
View File

@ -228,7 +228,7 @@ func (m *Sealing) restartSectors(ctx context.Context) error {
}
for _, sector := range trackedSectors {
if err := m.sectors.Send(sector.SectorID, SectorRestart{}); err != nil {
if err := m.sectors.Send(uint64(sector.SectorID), SectorRestart{}); err != nil {
log.Errorf("restarting sector %d: %+v", sector.SectorID, err)
}
}

102
storage/sealing/garbage.go
View File

@ -1,37 +1,25 @@
package sealing
import (
"bytes"
"context"
"io"
"math"
"math/bits"
"math/rand"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/market"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/specs-actors/actors/abi"
)
func (m *Sealing) pledgeReader(size abi.UnpaddedPieceSize, parts uint64) io.Reader {
parts = 1 << bits.Len64(parts) // round down to nearest power of 2
if uint64(size)/parts < 127 {
parts = uint64(size) / 127
type nullReader struct {}
func (nullReader) Read(out []byte) (int, error) {
for i := range out {
out[i] = 0
}
return len(out), nil
}
piece := abi.PaddedPieceSize(uint64(size.Padded()) / parts).Unpadded()
readers := make([]io.Reader, parts)
for i := range readers {
readers[i] = io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece))
}
return io.MultiReader(readers...)
func (m *Sealing) pledgeReader(size abi.UnpaddedPieceSize) io.Reader {
return io.LimitReader(&nullReader{}, int64(size))
}
func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, existingPieceSizes []abi.UnpaddedPieceSize, sizes ...abi.UnpaddedPieceSize) ([]Piece, error) {
@ -41,70 +29,9 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, e
log.Infof("Pledge %d, contains %+v", sectorID, existingPieceSizes)
deals := make([]market.ClientDealProposal, len(sizes))
for i, size := range sizes {
commP, err := m.fastPledgeCommitment(size, uint64(1))
if err != nil {
return nil, err
}
sdp := market.DealProposal{
PieceCID: commcid.PieceCommitmentV1ToCID(commP[:]),
PieceSize: size.Padded(),
Client: m.worker,
Provider: m.maddr,
StartEpoch: math.MaxInt64,
EndEpoch: math.MaxInt64,
StoragePricePerEpoch: types.NewInt(0),
ProviderCollateral: types.NewInt(0),
}
deals[i] = market.ClientDealProposal{
Proposal: sdp,
}
}
log.Infof("Publishing deals for %d", sectorID)
params, aerr := actors.SerializeParams(&market.PublishStorageDealsParams{
Deals: deals,
})
if aerr != nil {
return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", aerr)
}
smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress,
From: m.worker,
Value: types.NewInt(0),
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000),
Method: builtin.MethodsMarket.PublishStorageDeals,
Params: params,
})
if err != nil {
return nil, err
}
r, err := m.api.StateWaitMsg(ctx, smsg.Cid()) // TODO: more finality
if err != nil {
return nil, err
}
if r.Receipt.ExitCode != 0 {
log.Error(xerrors.Errorf("publishing deal (ts %s) %s failed: exit %d", r.TipSet.Key(), smsg.Cid(), r.Receipt.ExitCode))
}
var resp actors.PublishStorageDealResponse
if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil {
return nil, err
}
if len(resp.IDs) != len(sizes) {
return nil, xerrors.New("got unexpected number of DealIDs from PublishStorageDeals")
}
log.Infof("Deals for sector %d: %+v", sectorID, resp.IDs)
out := make([]Piece, len(sizes))
for i, size := range sizes {
ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size, uint64(1)), existingPieceSizes)
ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size), existingPieceSizes)
if err != nil {
return nil, xerrors.Errorf("add piece: %w", err)
}
@ -112,9 +39,8 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, e
existingPieceSizes = append(existingPieceSizes, size)
out[i] = Piece{
DealID: resp.IDs[i],
Size: abi.UnpaddedPieceSize(ppi.Size),
CommP: ppi.CommP[:],
Size: ppi.Size,
CommP: ppi.CommP[:],
}
}
@ -141,7 +67,7 @@ func (m *Sealing) PledgeSector() error {
return
}
if err := m.newSector(context.TODO(), sid, pieces[0].DealID, pieces[0].ppi()); err != nil {
if err := m.newSector(sid, pieces); err != nil {
log.Errorf("%+v", err)
return
}

22
storage/sealing/sealing.go
View File

@ -120,20 +120,20 @@ func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r i
return xerrors.Errorf("adding piece to sector: %w", err)
}
return m.newSector(ctx, sectorID, dealID, ppi)
return m.newSector(sectorID, []Piece{
{
DealID: &dealID,
Size: ppi.Size,
CommP: ppi.CommP[:],
},
},)
}
func (m *Sealing) newSector(ctx context.Context, sid abi.SectorNumber, dealID abi.DealID, ppi sectorbuilder.PublicPieceInfo) error {
func (m *Sealing) newSector(sid abi.SectorNumber, pieces []Piece) error {
log.Infof("Start sealing %d", sid)
return m.sectors.Send(sid, SectorStart{
return m.sectors.Send(uint64(sid), SectorStart{
id: sid,
pieces: []Piece{
{
DealID: dealID,
Size: abi.UnpaddedPieceSize(ppi.Size),
CommP: ppi.CommP[:],
},
},
pieces: pieces,
})
}

3
storage/sealing/states.go
View File

@ -99,8 +99,9 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
}
params := &miner.SectorPreCommitInfo{
Expiration: 0,
Expiration: 10000000, // TODO: implement
SectorNumber: sector.SectorID,
RegisteredProof: abi.RegisteredProof_StackedDRG32GiBSeal,
SealedCID: commcid.ReplicaCommitmentV1ToCID(sector.CommR),
SealRandEpoch: sector.Ticket.BlockHeight,

11
storage/sealing/types.go
View File

@ -34,7 +34,7 @@ func (t *SealSeed) Equals(o *SealSeed) bool {
}
type Piece struct {
DealID abi.DealID
DealID *abi.DealID
Size abi.UnpaddedPieceSize
CommP []byte
@ -97,9 +97,12 @@ func (t *SectorInfo) pieceInfos() []sectorbuilder.PublicPieceInfo {
}
func (t *SectorInfo) deals() []abi.DealID {
out := make([]abi.DealID, len(t.Pieces))
for i, piece := range t.Pieces {
out[i] = piece.DealID
out := make([]abi.DealID, 0, len(t.Pieces))
for _, piece := range t.Pieces {
if piece.DealID == nil {
continue
}
out = append(out, *piece.DealID)
}
return out
}

49
storage/sealing/utils.go
View File

@ -1,15 +1,8 @@
package sealing
import (
"io"
"math/bits"
"math/rand"
"sync"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/hashicorp/go-multierror"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"math/bits"
)
func fillersFromRem(in abi.UnpaddedPieceSize) ([]abi.UnpaddedPieceSize, error) {
@ -48,44 +41,6 @@ func fillersFromRem(in abi.UnpaddedPieceSize) ([]abi.UnpaddedPieceSize, error) {
return out, nil
}
func (m *Sealing) fastPledgeCommitment(size abi.UnpaddedPieceSize, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) {
parts = 1 << bits.Len64(parts) // round down to nearest power of 2
if uint64(size)/parts < 127 {
parts = uint64(size) / 127
}
piece := abi.PaddedPieceSize(uint64(size.Padded()) / parts).Unpadded()
out := make([]sectorbuilder.PublicPieceInfo, parts)
var lk sync.Mutex
var wg sync.WaitGroup
wg.Add(int(parts))
for i := uint64(0); i < parts; i++ {
go func(i uint64) {
defer wg.Done()
commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)), piece)
lk.Lock()
if perr != nil {
err = multierror.Append(err, perr)
}
out[i] = sectorbuilder.PublicPieceInfo{
Size: piece,
CommP: commP,
}
lk.Unlock()
}(i)
}
wg.Wait()
if err != nil {
return [32]byte{}, err
}
return sectorbuilder.GenerateDataCommitment(m.sb.SectorSize(), out)
}
func (m *Sealing) ListSectors() ([]SectorInfo, error) {
var sectors []SectorInfo
if err := m.sectors.List(&sectors); err != nil {
@ -96,6 +51,6 @@ func (m *Sealing) ListSectors() ([]SectorInfo, error) {
func (m *Sealing) GetSectorInfo(sid abi.SectorNumber) (SectorInfo, error) {
var out SectorInfo
err := m.sectors.Get(sid).Get(&out)
err := m.sectors.Get(uint64(sid)).Get(&out)
return out, err
}