diff --git a/.circleci/config.yml b/.circleci/config.yml index 73ed4dd7d..c8130f7a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,8 @@ commands: prepare: steps: - checkout + - run: sudo apt-get update + - run: sudo apt-get install ocl-icd-opencl-dev - run: git submodule sync - run: git submodule update --init download-params: @@ -23,14 +25,14 @@ commands: - restore_cache: name: Restore parameters cache keys: - - 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}' - - 'v2-lotus-params-' + - 'v15-lotus-params-{{ checksum "build/proof-params/parameters.json" }}' + - 'v15-lotus-params-' paths: - /var/tmp/filecoin-proof-parameters/ - run: ./lotus fetch-params - save_cache: name: Save parameters cache - key: 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}' + key: 'v15-lotus-params-{{ checksum "build/proof-params/parameters.json" }}' paths: - /var/tmp/filecoin-proof-parameters/ diff --git a/README.md b/README.md index c3c710864..2a4cb394c 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,35 @@ In order to run lotus, please do the following: - bzr (some go dependency needs this) - jq - pkg-config +- opencl-icd-loader +- opencl driver (like nvidia-opencl on arch) (for GPU acceleration) +- opencl-headers (build) +- rustup (proofs build) +- llvm (proofs build) +- clang (proofs build) +Arch (run): +```sh +sudo pacman -Syu opencl-icd-loader +``` + +Arch (build): +```sh +sudo pacman -Syu go gcc git bzr jq pkg-config opencl-icd-loader opencl-headers +``` + +Ubuntu / Debian (run): +```sh +sudo apt update +sudo apt install mesa-opencl-icd ocl-icd-opencl-dev +``` + +Ubuntu (build): +```sh +sudo add-apt-repository ppa:longsleep/golang-backports +sudo apt update +sudo apt install golang-go gcc git bzr jq pkg-config mesa-opencl-icd ocl-icd-opencl-dev +``` 2. Clone this repo & `cd` into it ``` diff --git a/api/api_common.go b/api/api_common.go new file mode 100644 index 000000000..c83a4c260 --- /dev/null +++ b/api/api_common.go @@ -0,0 +1,52 @@ +package api + +import ( + "context" + "fmt" + + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + + "github.com/filecoin-project/lotus/build" +) + +type Common interface { + // Auth + AuthVerify(ctx context.Context, token string) ([]Permission, error) + AuthNew(ctx context.Context, perms []Permission) ([]byte, error) + + // network + + NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) + NetPeers(context.Context) ([]peer.AddrInfo, error) + NetConnect(context.Context, peer.AddrInfo) error + NetAddrsListen(context.Context) (peer.AddrInfo, error) + NetDisconnect(context.Context, peer.ID) error + + // ID returns peerID of libp2p node backing this API + ID(context.Context) (peer.ID, error) + + // Version provides information about API provider + Version(context.Context) (Version, error) +} + +// Version provides various build-time information +type Version struct { + Version string + + // APIVersion is a binary encoded semver version of the remote implementing + // this api + // + // See APIVersion in build/version.go + APIVersion uint32 + + // TODO: git commit / os / genesis cid? + + // Seconds + BlockDelay uint64 +} + +func (v Version) String() string { + vM, vm, vp := build.VersionInts(v.APIVersion) + return fmt.Sprintf("%s+api%d.%d.%d", v.Version, vM, vm, vp) +} diff --git a/api/api.go b/api/api_full.go similarity index 79% rename from api/api.go rename to api/api_full.go index 463e5e9cb..057dca8cb 100644 --- a/api/api.go +++ b/api/api_full.go @@ -2,46 +2,17 @@ package api import ( "context" - "fmt" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/ipfs/go-filestore" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) -func init() { - cbor.RegisterCborType(SealedRef{}) -} - -type Common interface { - // Auth - AuthVerify(ctx context.Context, token string) ([]Permission, error) - AuthNew(ctx context.Context, perms []Permission) ([]byte, error) - - // network - - NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) - NetPeers(context.Context) ([]peer.AddrInfo, error) - NetConnect(context.Context, peer.AddrInfo) error - NetAddrsListen(context.Context) (peer.AddrInfo, error) - NetDisconnect(context.Context, peer.ID) error - - // ID returns peerID of libp2p node backing this API - ID(context.Context) (peer.ID, error) - - // Version provides information about API provider - Version(context.Context) (Version, error) -} - // FullNode API is a low-level interface to the Filecoin network full node type FullNode interface { Common @@ -103,9 +74,10 @@ type FullNode interface { // ClientImport imports file under the specified path into filestore ClientImport(ctx context.Context, path string) (cid.Cid, error) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, epochPrice types.BigInt, blocksDuration uint64) (*cid.Cid, error) + ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error) ClientListDeals(ctx context.Context) ([]DealInfo, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) - ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) + ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) @@ -123,8 +95,8 @@ type FullNode interface { StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error) - StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) - StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) + StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) + StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) StateMinerPower(context.Context, address.Address, *types.TipSet) (MinerPower, error) StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error) StateMinerPeerID(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) @@ -137,6 +109,10 @@ type FullNode interface { StateMarketBalance(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) StateMarketParticipants(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) StateMarketDeals(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) + StateMarketStorageDeal(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) + + MarketEnsureAvailable(context.Context, address.Address, types.BigInt) error + // MarketFreeBalance PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) PaychList(context.Context) ([]address.Address, error) @@ -152,47 +128,6 @@ type FullNode interface { PaychVoucherSubmit(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error) } -// StorageMiner is a low-level interface to the Filecoin network storage miner node -type StorageMiner interface { - Common - - ActorAddress(context.Context) (address.Address, error) - - // Temp api for testing - StoreGarbageData(context.Context) error - - // Get the status of a given sector by ID - SectorsStatus(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) - - // List all staged sectors - SectorsList(context.Context) ([]uint64, error) - - SectorsRefs(context.Context) (map[string][]SealedRef, error) - - CommitmentsList(context.Context) ([]SectorCommitment, error) -} - -// Version provides various build-time information -type Version struct { - Version string - - // APIVersion is a binary encoded semver version of the remote implementing - // this api - // - // See APIVersion in build/version.go - APIVersion uint32 - - // TODO: git commit / os / genesis cid? - - // Seconds - BlockDelay uint64 -} - -func (v Version) String() string { - vM, vm, vp := build.VersionInts(v.APIVersion) - return fmt.Sprintf("%s+api%d.%d.%d", v.Version, vM, vm, vp) -} - type Import struct { Status filestore.Status Key cid.Cid @@ -229,7 +164,7 @@ type Message struct { Message *types.Message } -type SectorInfo struct { +type ChainSectorInfo struct { SectorID uint64 CommD []byte CommR []byte @@ -277,12 +212,6 @@ type MinerPower struct { TotalPower types.BigInt } -type SealedRef struct { - Piece string - Offset uint64 - Size uint32 -} - type QueryOffer struct { Err string @@ -332,14 +261,6 @@ type SyncState struct { Height uint64 } -type SectorCommitment struct { - SectorID uint64 - Miner address.Address - - CommitMsg cid.Cid - DealIDs []uint64 -} - type SyncStateStage int const ( diff --git a/api/api_storage.go b/api/api_storage.go new file mode 100644 index 000000000..299449662 --- /dev/null +++ b/api/api_storage.go @@ -0,0 +1,96 @@ +package api + +import ( + "context" + "fmt" + + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/lib/sectorbuilder" +) + +// alias because cbor-gen doesn't like non-alias types +type SectorState = uint64 + +const ( + UndefinedSectorState SectorState = iota + + Empty // TODO: Is this useful + Packing // sector not in sealStore, and not on chain + + Unsealed // sealing / queued + PreCommitting // on chain pre-commit + PreCommitted // waiting for seed + Committing + Proving + + SectorNoUpdate = UndefinedSectorState +) + +func SectorStateStr(s SectorState) string { + switch s { + case UndefinedSectorState: + return "UndefinedSectorState" + case Empty: + return "Empty" + case Packing: + return "Packing" + case Unsealed: + return "Unsealed" + case PreCommitting: + return "PreCommitting" + case PreCommitted: + return "PreCommitted" + case Committing: + return "Committing" + case Proving: + return "Proving" + } + return fmt.Sprintf("", s) +} + +// StorageMiner is a low-level interface to the Filecoin network storage miner node +type StorageMiner interface { + Common + + ActorAddress(context.Context) (address.Address, error) + + // Temp api for testing + StoreGarbageData(context.Context) error + + // Get the status of a given sector by ID + SectorsStatus(context.Context, uint64) (SectorInfo, error) + + // List all staged sectors + SectorsList(context.Context) ([]uint64, error) + + SectorsRefs(context.Context) (map[string][]SealedRef, error) + + WorkerStats(context.Context) (WorkerStats, error) +} + +type WorkerStats struct { + Free int + Reserved int // for PoSt + Total int +} + +type SectorInfo struct { + SectorID uint64 + State SectorState + CommD []byte + CommR []byte + Proof []byte + Deals []uint64 + Ticket sectorbuilder.SealTicket + Seed sectorbuilder.SealSeed +} + +type SealedRef struct { + Piece string + Offset uint64 + Size uint64 +} + +type SealedRefs struct { + Refs []SealedRef +} diff --git a/api/cbor_gen.go b/api/cbor_gen.go new file mode 100644 index 000000000..0f968865b --- /dev/null +++ b/api/cbor_gen.go @@ -0,0 +1,271 @@ +package api + +import ( + "fmt" + "io" + + "github.com/filecoin-project/lotus/chain/types" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Channel (address.Address) (struct) + if err := t.Channel.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ChannelMessage (cid.Cid) (struct) + + if t.ChannelMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.ChannelMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.ChannelMessage: %w", err) + } + } + + // t.t.Vouchers ([]*types.SignedVoucher) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil { + return err + } + for _, v := range t.Vouchers { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + return nil +} + +func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Channel (address.Address) (struct) + + { + + if err := t.Channel.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ChannelMessage (cid.Cid) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err) + } + + t.ChannelMessage = &c + } + + } + // t.t.Vouchers ([]*types.SignedVoucher) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Vouchers: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Vouchers = make([]*types.SignedVoucher, extra) + } + for i := 0; i < int(extra); i++ { + + var v types.SignedVoucher + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Vouchers[i] = &v + } + + return nil +} + +func (t *SealedRef) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Piece (string) (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Piece)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Piece)); err != nil { + return err + } + + // t.t.Offset (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Offset))); err != nil { + return err + } + + // t.t.Size (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { + return err + } + return nil +} + +func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Piece (string) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Piece = string(sval) + } + // t.t.Offset (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Offset = uint64(extra) + // t.t.Size (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + return nil +} + +func (t *SealedRefs) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Refs ([]api.SealedRef) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Refs)))); err != nil { + return err + } + for _, v := range t.Refs { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + return nil +} + +func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Refs ([]api.SealedRef) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Refs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Refs = make([]SealedRef, extra) + } + for i := 0; i < int(extra); i++ { + + var v SealedRef + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Refs[i] = v + } + + return nil +} diff --git a/api/struct.go b/api/struct.go index 47d88aa69..8d14af1d2 100644 --- a/api/struct.go +++ b/api/struct.go @@ -3,7 +3,6 @@ package api import ( "context" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -82,12 +81,13 @@ type FullNodeStruct struct { ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` + ClientGetDealInfo func(context.Context, cid.Cid) (*DealInfo, error) `perm:"read"` ClientListDeals func(ctx context.Context) ([]DealInfo, error) `perm:"write"` ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"` - StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"` + StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"` StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"` @@ -104,6 +104,9 @@ type FullNodeStruct struct { StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"` StateMarketParticipants func(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) `perm:"read"` StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"` + StateMarketStorageDeal func(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) `perm:"read"` + + MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"` PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) `perm:"sign"` PaychList func(context.Context) ([]address.Address, error) `perm:"read"` @@ -129,12 +132,11 @@ type StorageMinerStruct struct { StoreGarbageData func(context.Context) error `perm:"write"` - SectorsStatus func(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) `perm:"read"` - SectorsList func(context.Context) ([]uint64, error) `perm:"read"` + SectorsStatus func(context.Context, uint64) (SectorInfo, error) `perm:"read"` + SectorsList func(context.Context) ([]uint64, error) `perm:"read"` + SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` - SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` - - CommitmentsList func(context.Context) ([]SectorCommitment, error) `perm:"read"` + WorkerStats func(context.Context) (WorkerStats, error) `perm:"read"` } } @@ -195,6 +197,9 @@ func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]Qu func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) { return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration) } +func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*DealInfo, error) { + return c.Internal.ClientGetDealInfo(ctx, deal) +} func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]DealInfo, error) { return c.Internal.ClientListDeals(ctx) @@ -340,11 +345,11 @@ func (c *FullNodeStruct) SyncSubmitBlock(ctx context.Context, blk *types.BlockMs return c.Internal.SyncSubmitBlock(ctx, blk) } -func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*SectorInfo, error) { +func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) { return c.Internal.StateMinerSectors(ctx, addr, ts) } -func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*SectorInfo, error) { +func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) { return c.Internal.StateMinerProvingSet(ctx, addr, ts) } @@ -411,6 +416,14 @@ func (c *FullNodeStruct) StateMarketDeals(ctx context.Context, ts *types.TipSet) return c.Internal.StateMarketDeals(ctx, ts) } +func (c *FullNodeStruct) StateMarketStorageDeal(ctx context.Context, dealid uint64, ts *types.TipSet) (*actors.OnChainDeal, error) { + return c.Internal.StateMarketStorageDeal(ctx, dealid, ts) +} + +func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error { + return c.Internal.MarketEnsureAvailable(ctx, addr, amt) +} + func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) { return c.Internal.PaychGet(ctx, from, to, ensureFunds) } @@ -468,7 +481,7 @@ func (c *StorageMinerStruct) StoreGarbageData(ctx context.Context) error { } // Get the status of a given sector by ID -func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (sectorbuilder.SectorSealingStatus, error) { +func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (SectorInfo, error) { return c.Internal.SectorsStatus(ctx, sid) } @@ -481,8 +494,8 @@ func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]Seal return c.Internal.SectorsRefs(ctx) } -func (c *StorageMinerStruct) CommitmentsList(ctx context.Context) ([]SectorCommitment, error) { - return c.Internal.CommitmentsList(ctx) +func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (WorkerStats, error) { + return c.Internal.WorkerStats(ctx) } var _ Common = &CommonStruct{} diff --git a/api/test/deals.go b/api/test/deals.go new file mode 100644 index 000000000..cb3002459 --- /dev/null +++ b/api/test/deals.go @@ -0,0 +1,96 @@ +package test + +import ( + "context" + "fmt" + "io" + "math/rand" + "os" + "testing" + "time" + + logging "github.com/ipfs/go-log" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/impl" +) + +func TestDealFlow(t *testing.T, b APIBuilder) { + os.Setenv("BELLMAN_NO_GPU", "1") + + logging.SetAllLoggers(logging.LevelInfo) + ctx := context.Background() + n, sn := b(t, 1, []int{0}) + client := n[0].FullNode.(*impl.FullNodeAPI) + miner := sn[0] + + addrinfo, err := client.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + + if err := miner.NetConnect(ctx, addrinfo); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + + r := io.LimitReader(rand.New(rand.NewSource(17)), 1000) + fcid, err := client.ClientImportLocal(ctx, r) + if err != nil { + t.Fatal(err) + } + + maddr, err := address.NewFromString("t0101") + if err != nil { + t.Fatal(err) + } + + fmt.Println("FILE CID: ", fcid) + + mine := true + done := make(chan struct{}) + + go func() { + defer close(done) + for mine { + time.Sleep(time.Second) + fmt.Println("mining a block now") + if err := n[0].MineOne(ctx); err != nil { + t.Fatal(err) + } + } + }() + deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(40000000), 100) + if err != nil { + t.Fatal(err) + } + + // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this + time.Sleep(time.Second) +loop: + for { + di, err := client.ClientGetDealInfo(ctx, *deal) + if err != nil { + t.Fatal(err) + } + switch di.State { + case api.DealRejected: + t.Fatal("deal rejected") + case api.DealFailed: + t.Fatal("deal failed") + case api.DealError: + t.Fatal("deal errored") + case api.DealComplete: + fmt.Println("COMPLETE", di) + break loop + } + fmt.Println("Deal state: ", api.DealStates[di.State]) + time.Sleep(time.Second / 2) + } + + mine = false + fmt.Println("shutting down mining") + <-done +} diff --git a/api/types.go b/api/types.go index 6a5153328..cd2717bbd 100644 --- a/api/types.go +++ b/api/types.go @@ -25,6 +25,17 @@ const ( DealNoUpdate = DealUnknown ) +var DealStates = []string{ + "DealUnknown", + "DealRejected", + "DealAccepted", + "DealStaged", + "DealSealing", + "DealFailed", + "DealComplete", + "DealError", +} + // TODO: check if this exists anywhere else type MultiaddrSlice []ma.Multiaddr diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 0319a6d97..4d803d299 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -1 +1 @@ -/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWPTYfTi7B1mQpbyRaCT65Trf7zW9npmihcF1xPfZogrzh +/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWLM9S94b9mnHD2fbWvQzXudoapHjbBbFA1T4QVft65Kym diff --git a/build/bootstrap/root.pi b/build/bootstrap/root.pi index b03183ddf..80ed9013a 100644 --- a/build/bootstrap/root.pi +++ b/build/bootstrap/root.pi @@ -1 +1 @@ -/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWEwGkxuG4BSvWUU8Sf69rxUYjWHAMT2qFVBhzGJZAXc6v +/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWB4pVpuRAcW2873V6m3uPWevKcZ1VWr1FGPUTwDZUN5mg diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car index ea19dc0d7..7d400c111 100644 Binary files a/build/genesis/devnet.car and b/build/genesis/devnet.car differ diff --git a/build/paramfetch.go b/build/paramfetch.go index 39e6ffda6..55dfd62bc 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -91,6 +91,11 @@ func (ft *fetch) maybeFetchAsync(name string, info paramFile) { } func (ft *fetch) checkFile(path string, info paramFile) error { + if os.Getenv("TRUST_PARAMS") == "1" { + log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION") + return nil + } + f, err := os.Open(path) if err != nil { return err diff --git a/build/params.go b/build/params.go index aaa7b475e..75f49e74d 100644 --- a/build/params.go +++ b/build/params.go @@ -46,16 +46,21 @@ const AllowableClockDrift = BlockDelay * 2 const ForkLengthThreshold = 100 // Blocks (e) -const BlocksPerEpoch = 3 +const BlocksPerEpoch = 5 // Blocks const Finality = 500 +// constants for Weight calculation +// The ratio of weight contributed by short-term vs long-term factors in a given round +const WRatioNum = int64(1) +const WRatioDen = 2 + // ///// // Proofs // Blocks -const ProvingPeriodDuration = 60 +const ProvingPeriodDuration = 160 // PoStChallangeTime sets the window in which post computation should happen // Blocks @@ -71,6 +76,9 @@ const PoStRandomnessLookback = 1 // Blocks const SealRandomnessLookback = Finality +// Blocks +const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000 + // ///// // Mining @@ -81,6 +89,9 @@ const PowerCollateralProportion = 5 const PerCapitaCollateralProportion = 1 const CollateralPrecision = 1000 +// Blocks +const InteractivePoRepDelay = 10 + // ///// // Devnet settings @@ -97,9 +108,6 @@ const FilecoinPrecision = 1_000_000_000_000_000_000 // Blocks const HalvingPeriodBlocks = 6 * 365 * 24 * 60 * 2 -// Blocks -const AdjustmentPeriod = 7 * 24 * 60 * 2 - // TODO: Move other important consts here func init() { diff --git a/build/proof-params/parameters.json b/build/proof-params/parameters.json index 0460fb226..e12686127 100644 --- a/build/proof-params/parameters.json +++ b/build/proof-params/parameters.json @@ -1,82 +1,82 @@ { - "v14-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": { + "v15-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": { "cid": "QmT22f1Np1GpW29NXD7Zrv3Ae4poMYhmkDjyscqL8QrJXY", "digest": "989fd8d989e0f7f1fe21bb010cf1b231", "sector_size": 16777216 }, - "v14-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": { + "v15-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": { "cid": "QmVqSdc23to4UwduCCb25223rpSccvtcgPMfRKY1qjucDc", "digest": "c6d258c37243b8544238a98100e3e399", "sector_size": 16777216 }, - "v14-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": { + "v15-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": { "cid": "QmRTCqgokEGTMfWVaSr7qFXTNotmpd2QBEi8RsvSQKmPLz", "digest": "ff77a5e270afc6e1c7fbc19e48348fac", "sector_size": 1073741824 }, - "v14-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": { + "v15-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": { "cid": "QmRssVAXRN3xp9VdSpTq1pNjkob3QiikoFZiM5hqrmh1VU", "digest": "b41f35ac26224258e366327716a835a4", "sector_size": 1073741824 }, - "v14-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": { + "v15-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": { "cid": "QmYNVRVzjXkuxJfnHTU5vmEcUBQf8dabXZ4m53SzqMkBv5", "digest": "d156b685e4a1fe3a1f7230b6a39b5ad4", "sector_size": 1024 }, - "v14-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": { + "v15-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": { "cid": "QmaCEcsCFVuepMKdC5WURbr5ucEyLMNGxQaB7HqSnr2KGh", "digest": "06ff067ac78cdab5d7bbc82170882241", "sector_size": 1024 }, - "v14-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": { + "v15-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": { "cid": "QmVuabRvJ797NwLisGKwRURASGxopBBgg4rfNsbZoSYzAc", "digest": "0e1ceb79a459a60508f480e5b1fed7ac", "sector_size": 268435456 }, - "v14-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": { + "v15-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": { "cid": "QmdWENZBAbuUty1vVNn9vmvj1XbJ5UC8qzpcVD35s5AJxG", "digest": "1b755c74b9d6823c014f6a7ef76249f2", "sector_size": 268435456 }, - "v14-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.params": { - "cid": "QmPG38HmDNVFiQJskqKe9sfSjyHfvZRihcfry78rt22FDT", - "digest": "8ea0b47e72250d5d6dab5d4f859e65de", + "v15-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.params": { + "cid": "QmZDVpWTw5Eti5pE7N5z1Cmqsw8hPXhUcvG3cQuceK56LH", + "digest": "6aa80306018ea1328f2d6faf8c080734", "sector_size": 16777216 }, - "v14-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.vk": { - "cid": "Qmd3pNM22pgAoRT24tNyEZmeEWK2GtoZznBvzjie2YgqCn", - "digest": "e39f344757c919ae6bbc9b61311c73b2", + "v15-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.vk": { + "cid": "QmaoXV7iVSJcfZ5qubYy7NBcXDSdnTzxH85d7M4bdDtfGZ", + "digest": "f6832eb736faf2960e920d32e9780b12", "sector_size": 16777216 }, - "v14-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.params": { - "cid": "QmQsS6RqWmgdwPnHCwhBJH3WDPcAxhKfbQUs2bwa8D9su8", - "digest": "09879a69abcc51de5c1095f347c84e2b", + "v15-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.params": { + "cid": "QmbUW3a3q5DHBb7Ufk8iSbnSCZgbwpe3serqfwKmcTd11w", + "digest": "64024e461b07c869df2463d33dd28035", "sector_size": 268435456 }, - "v14-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.vk": { - "cid": "QmSFBL5rg2TJv8QjLzrbr4c2KV2uDNN13RBVNUgwemJgM1", - "digest": "db0f245f7e9989879d2fa6328bd57d32", + "v15-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.vk": { + "cid": "Qme3QgBBE7hUgrK7G9ZfJhzkbvViN5HALFpFduYs5K1piv", + "digest": "32496f4dc434b0ed9ef49cb62497a7d1", "sector_size": 268435456 }, - "v14-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.params": { - "cid": "QmZ9UVBfviaNFsKyazA4k8GZcM1tHNDpDyrEK9qkxaMJXx", - "digest": "402a7c7c82eaa4af9fba3c7e4402b65b", + "v15-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.params": { + "cid": "QmZzgJmb8WXYDKxS22HDgnoBYcZzXDC7s2c2zsV7kouNZ9", + "digest": "cd91f7ccb2ff57a06f3375946dcbdc68", "sector_size": 1073741824 }, - "v14-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.vk": { - "cid": "QmNmDcPVJ1bFFDcNCcRnEoQ6vNDNpKadLPHyEpUoF47gxV", - "digest": "2741c456346a3758e88249d1f4c0d227", + "v15-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.vk": { + "cid": "QmRUMVzFnENbvyNb6aN2AJ2dnnewr1ESGA1UQLMVZZdsJM", + "digest": "92fc84b76dbe69c731518aebcb82ac82", "sector_size": 1073741824 }, - "v14-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.params": { - "cid": "QmNqgCv6UjdKDNMuiwDweZ22TQYMd3gV6nWiA5PjMtNDVu", - "digest": "a9d316d0dbca152e653d41ad8e40058a", + "v15-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.params": { + "cid": "QmSixsGkxJXTAuFfWKy5aEXEDJEnpcb1GkdQVF8TCPWoHy", + "digest": "f8339ae93478ded3840d0bc7efa19953", "sector_size": 1024 }, - "v14-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.vk": { - "cid": "QmTGQGThNFEjaMFgP32YLubACrtpkRoeVEfhDaWi5g6w8u", - "digest": "021b3e81e2980a50fd1ac07424d29a8d", + "v15-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.vk": { + "cid": "QmTMC8hdZ2TkZ9BFuzHzRLM9SuR2PQdUrSAwABeCuHyV2f", + "digest": "f27f08ce1246ee6612c250bb12803ef1", "sector_size": 1024 } } \ No newline at end of file diff --git a/build/version.go b/build/version.go index 33109762d..abc715b48 100644 --- a/build/version.go +++ b/build/version.go @@ -1,7 +1,7 @@ package build // Version is the local build version, set by build system -const Version = "0.0.0" +const Version = "0.7.0" // APIVersion is a hex semver version of the rpc api exposed // @@ -12,7 +12,7 @@ const Version = "0.0.0" // R R H // |\vv/| // vv vv -const APIVersion = 0x000001 +const APIVersion = 0x000701 const ( MajorMask = 0xff0000 diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index baa35d0d5..d1e4ad4d3 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -2,6 +2,7 @@ package actors import ( "context" + "encoding/binary" "fmt" "github.com/filecoin-project/lotus/build" @@ -21,22 +22,27 @@ import ( type StorageMinerActor struct{} type StorageMinerActorState struct { - // Contains mostly static info about this miner - Info cid.Cid - - // Collateral that is waiting to be withdrawn. - DePledgedCollateral types.BigInt - - // Time at which the depledged collateral may be withdrawn. - DePledgeTime types.BigInt + // PreCommittedSectors is the set of sectors that have been committed to but not + // yet had their proofs submitted + PreCommittedSectors map[string]*PreCommittedSector // All sectors this miner has committed. Sectors cid.Cid + // TODO: Spec says 'StagedCommittedSectors', which one is it? + // Sectors this miner is currently mining. It is only updated // when a PoSt is submitted (not as each new sector commitment is added). ProvingSet cid.Cid + // TODO: these: + // SectorTable + // SectorExpirationQueue + // ChallengeStatus + + // Contains mostly static info about this miner + Info cid.Cid + // Faulty sectors reported since last SubmitPost, // up to the current proving period's challenge time. CurrentFaultSet types.BitField @@ -52,10 +58,6 @@ type StorageMinerActorState struct { // removal penalization is needed. NextDoneSet types.BitField - // Deals this miner has been slashed for since the last post submission. - //TODO: unsupported map key type "Cid" (if you want to use struct keys, your atlas needs a transform to string) - //ArbitratedDeals map[cid.Cid]struct{} - // Amount of power this miner has. Power types.BigInt @@ -88,6 +90,13 @@ type MinerInfo struct { // Amount of space in each sector committed to the network by this miner. SectorSize uint64 + + // SubsectorCount +} + +type PreCommittedSector struct { + Info SectorPreCommitInfo + ReceivedEpoch uint64 } type StorageMinerConstructorParams struct { @@ -97,9 +106,18 @@ type StorageMinerConstructorParams struct { PeerID peer.ID } +type SectorPreCommitInfo struct { + SectorNumber uint64 + + CommR []byte // TODO: Spec says CID + SealEpoch uint64 + DealIDs []uint64 +} + type maMethods struct { Constructor uint64 - CommitSector uint64 + PreCommitSector uint64 + ProveCommitSector uint64 SubmitPoSt uint64 SlashStorageFault uint64 GetCurrentProvingSet uint64 @@ -114,32 +132,33 @@ type maMethods struct { ChangeWorker uint64 IsSlashed uint64 IsLate uint64 - AddFaults uint64 + DeclareFaults uint64 SlashConsensusFault uint64 } -var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18} +var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19} func (sma StorageMinerActor) Exports() []interface{} { return []interface{}{ 1: sma.StorageMinerConstructor, - 2: sma.CommitSector, - 3: sma.SubmitPoSt, - //4: sma.SlashStorageFault, - //5: sma.GetCurrentProvingSet, - //6: sma.ArbitrateDeal, - //7: sma.DePledge, - 8: sma.GetOwner, - 9: sma.GetWorkerAddr, - 10: sma.GetPower, - 11: sma.GetPeerID, - 12: sma.GetSectorSize, - 13: sma.UpdatePeerID, - //14: sma.ChangeWorker, - //15: sma.IsSlashed, - //16: sma.IsLate, - 17: sma.AddFaults, - 18: sma.SlashConsensusFault, + 2: sma.PreCommitSector, + 3: sma.ProveCommitSector, + 4: sma.SubmitPoSt, + //5: sma.SlashStorageFault, + //6: sma.GetCurrentProvingSet, + //7: sma.ArbitrateDeal, + //8: sma.DePledge, + 9: sma.GetOwner, + 10: sma.GetWorkerAddr, + 11: sma.GetPower, // TODO: Remove + 12: sma.GetPeerID, + 13: sma.GetSectorSize, + 14: sma.UpdatePeerID, + //15: sma.ChangeWorker, + //16: sma.IsSlashed, + //17: sma.IsLate, + 18: sma.DeclareFaults, + 19: sma.SlashConsensusFault, } } @@ -199,23 +218,21 @@ func (sma StorageMinerActor) StorageMinerConstructor(act *types.Actor, vmctx typ return nil, nil } -type OnChainSealVerifyInfo struct { - CommD []byte // TODO: update proofs code - CommR []byte - - Epoch uint64 - Proof []byte - DealIDs []uint64 - SectorNumber uint64 -} - -func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContext, params *OnChainSealVerifyInfo) ([]byte, ActorError) { - ctx := context.TODO() +func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorPreCommitInfo) ([]byte, ActorError) { + ctx := vmctx.Context() oldstate, self, err := loadState(vmctx) if err != nil { return nil, err } + if params.SealEpoch >= vmctx.BlockHeight()+build.SealRandomnessLookback { + return nil, aerrors.Newf(1, "sector commitment must be based off past randomness (%d >= %d)", params.SealEpoch, vmctx.BlockHeight()+build.SealRandomnessLookback) + } + + if vmctx.BlockHeight()-params.SealEpoch+build.SealRandomnessLookback > build.SealRandomnessLookbackLimit { + return nil, aerrors.Newf(2, "sector commitment must be recent enough (was %d)", vmctx.BlockHeight()-params.SealEpoch+build.SealRandomnessLookback) + } + mi, err := loadMinerInfo(vmctx, self) if err != nil { return nil, err @@ -225,20 +242,6 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex return nil, aerrors.New(1, "not authorized to commit sector for miner") } - // TODO: this needs to get normalized to either the ID address or the actor address - maddr := vmctx.Message().To - - ticket, err := vmctx.GetRandomness(params.Epoch) - if err != nil { - return nil, aerrors.Wrap(err, "failed to get randomness for commitsector") - } - - if ok, err := ValidatePoRep(maddr, mi.SectorSize, params, ticket); err != nil { - return nil, err - } else if !ok { - return nil, aerrors.New(2, "bad proof!") - } - // make sure the miner isnt trying to submit a pre-existing sector unique, err := SectorIsUnique(ctx, vmctx.Storage(), self.Sectors, params.SectorNumber) if err != nil { @@ -257,10 +260,93 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex return nil, aerrors.New(4, "not enough collateral") } + self.PreCommittedSectors[uintToStringKey(params.SectorNumber)] = &PreCommittedSector{ + Info: *params, + ReceivedEpoch: vmctx.BlockHeight(), + } + + nstate, err := vmctx.Storage().Put(self) + if err != nil { + return nil, err + } + if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { + return nil, err + } + + return nil, nil +} + +func uintToStringKey(i uint64) string { + buf := make([]byte, 10) + n := binary.PutUvarint(buf, i) + return string(buf[:n]) +} + +type SectorProveCommitInfo struct { + Proof []byte + SectorID uint64 + DealIDs []uint64 +} + +func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) { + ctx := vmctx.Context() + oldstate, self, err := loadState(vmctx) + if err != nil { + return nil, err + } + + mi, err := loadMinerInfo(vmctx, self) + if err != nil { + return nil, err + } + + us, ok := self.PreCommittedSectors[uintToStringKey(params.SectorID)] + if !ok { + return nil, aerrors.New(1, "no pre-commitment found for sector") + } + + if us.ReceivedEpoch+build.InteractivePoRepDelay >= vmctx.BlockHeight() { + return nil, aerrors.New(2, "too early for proof submission") + } + + delete(self.PreCommittedSectors, uintToStringKey(params.SectorID)) + + // TODO: ensure normalization to ID address + maddr := vmctx.Message().To + + ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback) + if err != nil { + return nil, aerrors.Wrap(err, "failed to get ticket randomness") + } + + seed, err := vmctx.GetRandomness(us.ReceivedEpoch + build.InteractivePoRepDelay) + if err != nil { + return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment") + } + + enc, err := SerializeParams(&ComputeDataCommitmentParams{ + DealIDs: params.DealIDs, + SectorSize: mi.SectorSize, + }) + if err != nil { + return nil, aerrors.Wrap(err, "failed to serialize ComputeDataCommitmentParams") + } + + commD, err := vmctx.Send(StorageMarketAddress, SMAMethods.ComputeDataCommitment, types.NewInt(0), enc) + if err != nil { + return nil, aerrors.Wrap(err, "failed to compute data commitment") + } + + if ok, err := ValidatePoRep(maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { + return nil, err + } else if !ok { + return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%x)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, params.Proof) + } + // Note: There must exist a unique index in the miner's sector set for each // sector ID. The `faults`, `recovered`, and `done` parameters of the // SubmitPoSt method express indices into this sector set. - nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorNumber, params.CommR, params.CommD) + nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorID, us.Info.CommR, commD) if err != nil { return nil, err } @@ -270,7 +356,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex // Note: As written here, every miners first PoSt will only be over one sector. // We could set up a 'grace period' for starting mining that would allow miners // to submit several sectors for their first proving period. Alternatively, we - // could simply make the 'CommitSector' call take multiple sectors at a time. + // could simply make the 'PreCommitSector' call take multiple sectors at a time. // // Note: Proving period is a function of sector size; small sectors take less // time to prove than large sectors do. Sector size is selected when pledging. @@ -491,6 +577,8 @@ func AddToSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID u return cid.Undef, aerrors.HandleExternalError(err, "could not load sector set node") } + // TODO: Spec says to use SealCommitment, and construct commD from deals each time, + // but that would make SubmitPoSt way, way more expensive if err := ssr.Set(sectorID, [][]byte{commR, commD}); err != nil { return cid.Undef, aerrors.HandleExternalError(err, "failed to set commitment in sector set") } @@ -525,8 +613,8 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID return true, comms[0], comms[1], nil } -func ValidatePoRep(maddr address.Address, ssize uint64, params *OnChainSealVerifyInfo, ticket []byte) (bool, ActorError) { - ok, err := sectorbuilder.VerifySeal(ssize, params.CommR, params.CommD, maddr, ticket, params.SectorNumber, params.Proof) +func ValidatePoRep(maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) { + ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof) if err != nil { return false, aerrors.Absorb(err, 25, "verify seal failed") } @@ -648,11 +736,11 @@ type PaymentVerifyParams struct { Proof []byte } -type AddFaultsParams struct { +type DeclareFaultsParams struct { Faults types.BitField } -func (sma StorageMinerActor) AddFaults(act *types.Actor, vmctx types.VMContext, params *AddFaultsParams) ([]byte, ActorError) { +func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { oldstate, self, aerr := loadState(vmctx) if aerr != nil { return nil, aerr diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 9c3fcb9aa..6fb07be3f 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" ) type StorageMarketActor struct{} @@ -29,9 +30,10 @@ type smaMethods struct { SlashStorageDealCollateral uint64 GetLastExpirationFromDealIDs uint64 ActivateStorageDeals uint64 + ComputeDataCommitment uint64 } -var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11} +var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} func (sma StorageMarketActor) Exports() []interface{} { return []interface{}{ @@ -45,6 +47,7 @@ func (sma StorageMarketActor) Exports() []interface{} { // 9: sma.SlashStorageDealCollateral, // 10: sma.GetLastExpirationFromDealIDs, 11: sma.ActivateStorageDeals, // TODO: move under PublishStorageDeals after specs team approves + 12: sma.ComputeDataCommitment, } } @@ -433,7 +436,6 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) if err != nil { - // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal return nil, aerrors.HandleExternalError(err, "loading deals amt") } @@ -499,7 +501,6 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) if err != nil { - // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal return nil, aerrors.HandleExternalError(err, "loading deals amt") } @@ -585,6 +586,54 @@ func transferFunds(from, to, amt types.BigInt) (types.BigInt, types.BigInt) { return types.BigSub(from, amt), types.BigAdd(to, amt) } +type ComputeDataCommitmentParams struct { + DealIDs []uint64 + SectorSize uint64 +} + +func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx types.VMContext, params *ComputeDataCommitmentParams) ([]byte, ActorError) { + var self StorageMarketState + old := vmctx.Storage().GetHead() + if err := vmctx.Storage().Get(old, &self); err != nil { + return nil, err + } + + deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) + if err != nil { + return nil, aerrors.HandleExternalError(err, "loading deals amt") + } + + var pieces []sectorbuilder.PublicPieceInfo + for _, deal := range params.DealIDs { + var dealInfo OnChainDeal + if err := deals.Get(deal, &dealInfo); err != nil { + if _, is := err.(*amt.ErrNotFound); is { + return nil, aerrors.New(3, "deal not found") + } + return nil, aerrors.HandleExternalError(err, "getting deal info failed") + } + + if dealInfo.Deal.Proposal.Provider != vmctx.Message().From { + return nil, aerrors.New(4, "referenced deal was not from caller") + } + + var commP [32]byte + copy(commP[:], dealInfo.Deal.Proposal.PieceRef) + + pieces = append(pieces, sectorbuilder.PublicPieceInfo{ + Size: dealInfo.Deal.Proposal.PieceSize, + CommP: commP, + }) + } + + commd, err := sectorbuilder.GenerateDataCommitment(params.SectorSize, pieces) + if err != nil { + return nil, aerrors.Absorb(err, 5, "failed to generate data commitment from pieces") + } + + return commd[:], nil +} + /* func (sma StorageMarketActor) HandleCronAction(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { diff --git a/chain/actors/actor_storagepower.go b/chain/actors/actor_storagepower.go index ea564bc62..e3f7cbfd3 100644 --- a/chain/actors/actor_storagepower.go +++ b/chain/actors/actor_storagepower.go @@ -329,6 +329,7 @@ func powerLookup(ctx context.Context, vmctx types.VMContext, self *StoragePowerS return types.EmptyInt, aerrors.New(1, "miner not registered with storage power actor") } + // TODO: Use local amt ret, err := vmctx.Send(miner, MAMethods.GetPower, types.NewInt(0), nil) if err != nil { return types.EmptyInt, aerrors.Wrap(err, "invoke Miner.GetPower") diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 5444dafb0..fbb289fd2 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3,6 +3,7 @@ package actors import ( "fmt" "io" + "sort" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" @@ -24,14 +25,14 @@ func (t *InitActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.AddressMap (cid.Cid) + // t.t.AddressMap (cid.Cid) (struct) if err := cbg.WriteCid(w, t.AddressMap); err != nil { return xerrors.Errorf("failed to write cid field t.AddressMap: %w", err) } - // t.t.NextID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.NextID)); err != nil { + // t.t.NextID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextID))); err != nil { return err } return nil @@ -52,7 +53,7 @@ func (t *InitActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.AddressMap (cid.Cid) + // t.t.AddressMap (cid.Cid) (struct) { @@ -64,7 +65,7 @@ func (t *InitActorState) UnmarshalCBOR(r io.Reader) error { t.AddressMap = c } - // t.t.NextID (uint64) + // t.t.NextID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -73,7 +74,7 @@ func (t *InitActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.NextID = extra + t.NextID = uint64(extra) return nil } @@ -86,13 +87,13 @@ func (t *ExecParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Code (cid.Cid) + // t.t.Code (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Code); err != nil { return xerrors.Errorf("failed to write cid field t.Code: %w", err) } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Params)))); err != nil { return err } @@ -117,7 +118,7 @@ func (t *ExecParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Code (cid.Cid) + // t.t.Code (cid.Cid) (struct) { @@ -129,7 +130,7 @@ func (t *ExecParams) UnmarshalCBOR(r io.Reader) error { t.Code = c } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -158,7 +159,7 @@ func (t *AccountActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Address (address.Address) + // t.t.Address (address.Address) (struct) if err := t.Address.MarshalCBOR(w); err != nil { return err } @@ -180,7 +181,7 @@ func (t *AccountActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Address (address.Address) + // t.t.Address (address.Address) (struct) { @@ -197,70 +198,88 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{140}); err != nil { + if _, err := w.Write([]byte{139}); err != nil { return err } - // t.t.Info (cid.Cid) + // t.t.PreCommittedSectors (map[string]*actors.PreCommittedSector) (map) + { + if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.PreCommittedSectors))); err != nil { + return err + } - if err := cbg.WriteCid(w, t.Info); err != nil { - return xerrors.Errorf("failed to write cid field t.Info: %w", err) + keys := make([]string, 0, len(t.PreCommittedSectors)) + for k := range t.PreCommittedSectors { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := t.PreCommittedSectors[k] + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { + return err + } + if _, err := w.Write([]byte(k)); err != nil { + return err + } + + if err := v.MarshalCBOR(w); err != nil { + return err + } + + } } - // t.t.DePledgedCollateral (types.BigInt) - if err := t.DePledgedCollateral.MarshalCBOR(w); err != nil { - return err - } - - // t.t.DePledgeTime (types.BigInt) - if err := t.DePledgeTime.MarshalCBOR(w); err != nil { - return err - } - - // t.t.Sectors (cid.Cid) + // t.t.Sectors (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Sectors); err != nil { return xerrors.Errorf("failed to write cid field t.Sectors: %w", err) } - // t.t.ProvingSet (cid.Cid) + // t.t.ProvingSet (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ProvingSet); err != nil { return xerrors.Errorf("failed to write cid field t.ProvingSet: %w", err) } - // t.t.CurrentFaultSet (types.BitField) + // t.t.Info (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Info); err != nil { + return xerrors.Errorf("failed to write cid field t.Info: %w", err) + } + + // t.t.CurrentFaultSet (types.BitField) (struct) if err := t.CurrentFaultSet.MarshalCBOR(w); err != nil { return err } - // t.t.NextFaultSet (types.BitField) + // t.t.NextFaultSet (types.BitField) (struct) if err := t.NextFaultSet.MarshalCBOR(w); err != nil { return err } - // t.t.NextDoneSet (types.BitField) + // t.t.NextDoneSet (types.BitField) (struct) if err := t.NextDoneSet.MarshalCBOR(w); err != nil { return err } - // t.t.Power (types.BigInt) + // t.t.Power (types.BigInt) (struct) if err := t.Power.MarshalCBOR(w); err != nil { return err } - // t.t.SlashedAt (types.BigInt) + // t.t.SlashedAt (types.BigInt) (struct) if err := t.SlashedAt.MarshalCBOR(w); err != nil { return err } - // t.t.OwedStorageCollateral (types.BigInt) + // t.t.OwedStorageCollateral (types.BigInt) (struct) if err := t.OwedStorageCollateral.MarshalCBOR(w); err != nil { return err } - // t.t.ProvingPeriodEnd (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ProvingPeriodEnd)); err != nil { + // t.t.ProvingPeriodEnd (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProvingPeriodEnd))); err != nil { return err } return nil @@ -277,41 +296,64 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 12 { + if extra != 11 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Info (cid.Cid) + // t.t.PreCommittedSectors (map[string]*actors.PreCommittedSector) (map) - { + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("expected a map (major type 5)") + } + if extra > 4096 { + return fmt.Errorf("t.PreCommittedSectors: map too large") + } - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Info: %w", err) + t.PreCommittedSectors = make(map[string]*PreCommittedSector, extra) + + for i, l := 0, int(extra); i < l; i++ { + + var k string + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + k = string(sval) } - t.Info = c + var v *PreCommittedSector - } - // t.t.DePledgedCollateral (types.BigInt) + { - { + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + v = new(PreCommittedSector) + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + } - if err := t.DePledgedCollateral.UnmarshalCBOR(br); err != nil { - return err } - } - // t.t.DePledgeTime (types.BigInt) - - { - - if err := t.DePledgeTime.UnmarshalCBOR(br); err != nil { - return err - } + t.PreCommittedSectors[k] = v } - // t.t.Sectors (cid.Cid) + // t.t.Sectors (cid.Cid) (struct) { @@ -323,7 +365,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { t.Sectors = c } - // t.t.ProvingSet (cid.Cid) + // t.t.ProvingSet (cid.Cid) (struct) { @@ -335,7 +377,19 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { t.ProvingSet = c } - // t.t.CurrentFaultSet (types.BitField) + // t.t.Info (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Info: %w", err) + } + + t.Info = c + + } + // t.t.CurrentFaultSet (types.BitField) (struct) { @@ -344,7 +398,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.NextFaultSet (types.BitField) + // t.t.NextFaultSet (types.BitField) (struct) { @@ -353,7 +407,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.NextDoneSet (types.BitField) + // t.t.NextDoneSet (types.BitField) (struct) { @@ -362,7 +416,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Power (types.BigInt) + // t.t.Power (types.BigInt) (struct) { @@ -371,7 +425,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.SlashedAt (types.BigInt) + // t.t.SlashedAt (types.BigInt) (struct) { @@ -380,7 +434,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.OwedStorageCollateral (types.BigInt) + // t.t.OwedStorageCollateral (types.BigInt) (struct) { @@ -389,7 +443,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProvingPeriodEnd (uint64) + // t.t.ProvingPeriodEnd (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -398,7 +452,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ProvingPeriodEnd = extra + t.ProvingPeriodEnd = uint64(extra) return nil } @@ -411,22 +465,22 @@ func (t *StorageMinerConstructorParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) if err := t.Owner.MarshalCBOR(w); err != nil { return err } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) if err := t.Worker.MarshalCBOR(w); err != nil { return err } - // t.t.SectorSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { + // t.t.SectorSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorSize))); err != nil { return err } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.PeerID)))); err != nil { return err } @@ -451,7 +505,7 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) { @@ -460,7 +514,7 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) { @@ -469,7 +523,7 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.SectorSize (uint64) + // t.t.SectorSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -478,8 +532,8 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorSize = extra - // t.t.PeerID (peer.ID) + t.SectorSize = uint64(extra) + // t.t.PeerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -492,24 +546,21 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { +func (t *SectorPreCommitInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{134}); err != nil { + if _, err := w.Write([]byte{132}); err != nil { return err } - // t.t.CommD ([]uint8) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { - return err - } - if _, err := w.Write(t.CommD); err != nil { + // t.t.SectorNumber (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorNumber))); err != nil { return err } - // t.t.CommR ([]uint8) + // t.t.CommR ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { return err } @@ -517,20 +568,12 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Epoch (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Epoch)); err != nil { + // t.t.SealEpoch (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SealEpoch))); err != nil { return err } - // t.t.Proof ([]uint8) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { - return err - } - if _, err := w.Write(t.Proof); err != nil { - return err - } - - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { return err } @@ -539,15 +582,10 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return err } } - - // t.t.SectorNumber (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorNumber)); err != nil { - return err - } return nil } -func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { +func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -558,28 +596,21 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 6 { + if extra != 4 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.CommD ([]uint8) + // t.t.SectorNumber (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.CommD: array too large (%d)", extra) + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.CommD = make([]byte, extra) - if _, err := io.ReadFull(br, t.CommD); err != nil { - return err - } - // t.t.CommR ([]uint8) + t.SectorNumber = uint64(extra) + // t.t.CommR ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -596,7 +627,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.CommR); err != nil { return err } - // t.t.Epoch (uint64) + // t.t.SealEpoch (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -605,25 +636,8 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Epoch = extra - // t.t.Proof ([]uint8) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Proof = make([]byte, extra) - if _, err := io.ReadFull(br, t.Proof); err != nil { - return err - } - // t.t.DealIDs ([]uint64) + t.SealEpoch = uint64(extra) + // t.t.DealIDs ([]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -653,7 +667,55 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { t.DealIDs[i] = val } - // t.t.SectorNumber (uint64) + return nil +} + +func (t *PreCommittedSector) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Info (actors.SectorPreCommitInfo) (struct) + if err := t.Info.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ReceivedEpoch (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ReceivedEpoch))); err != nil { + return err + } + return nil +} + +func (t *PreCommittedSector) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Info (actors.SectorPreCommitInfo) (struct) + + { + + if err := t.Info.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ReceivedEpoch (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -662,7 +724,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorNumber = extra + t.ReceivedEpoch = uint64(extra) return nil } @@ -675,17 +737,17 @@ func (t *MinerInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) if err := t.Owner.MarshalCBOR(w); err != nil { return err } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) if err := t.Worker.MarshalCBOR(w); err != nil { return err } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.PeerID)))); err != nil { return err } @@ -693,8 +755,8 @@ func (t *MinerInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.SectorSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { + // t.t.SectorSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorSize))); err != nil { return err } return nil @@ -715,7 +777,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) { @@ -724,7 +786,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) { @@ -733,7 +795,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -743,7 +805,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { t.PeerID = peer.ID(sval) } - // t.t.SectorSize (uint64) + // t.t.SectorSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -752,7 +814,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorSize = extra + t.SectorSize = uint64(extra) return nil } @@ -765,7 +827,7 @@ func (t *SubmitPoStParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -773,7 +835,7 @@ func (t *SubmitPoStParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.DoneSet (types.BitField) + // t.t.DoneSet (types.BitField) (struct) if err := t.DoneSet.MarshalCBOR(w); err != nil { return err } @@ -795,7 +857,7 @@ func (t *SubmitPoStParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -812,7 +874,7 @@ func (t *SubmitPoStParams) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Proof); err != nil { return err } - // t.t.DoneSet (types.BitField) + // t.t.DoneSet (types.BitField) (struct) { @@ -833,7 +895,7 @@ func (t *PaymentVerifyParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Extra ([]uint8) + // t.t.Extra ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Extra)))); err != nil { return err } @@ -841,7 +903,7 @@ func (t *PaymentVerifyParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -866,7 +928,7 @@ func (t *PaymentVerifyParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Extra ([]uint8) + // t.t.Extra ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -883,7 +945,7 @@ func (t *PaymentVerifyParams) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Extra); err != nil { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -912,7 +974,7 @@ func (t *UpdatePeerIDParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.PeerID)))); err != nil { return err } @@ -937,7 +999,7 @@ func (t *UpdatePeerIDParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -959,7 +1021,7 @@ func (t *MultiSigActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Signers ([]address.Address) + // t.t.Signers ([]address.Address) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Signers)))); err != nil { return err } @@ -969,32 +1031,32 @@ func (t *MultiSigActorState) MarshalCBOR(w io.Writer) error { } } - // t.t.Required (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Required)); err != nil { + // t.t.Required (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Required))); err != nil { return err } - // t.t.NextTxID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.NextTxID)); err != nil { + // t.t.NextTxID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextTxID))); err != nil { return err } - // t.t.InitialBalance (types.BigInt) + // t.t.InitialBalance (types.BigInt) (struct) if err := t.InitialBalance.MarshalCBOR(w); err != nil { return err } - // t.t.StartingBlock (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.StartingBlock)); err != nil { + // t.t.StartingBlock (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.StartingBlock))); err != nil { return err } - // t.t.UnlockDuration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.UnlockDuration)); err != nil { + // t.t.UnlockDuration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.UnlockDuration))); err != nil { return err } - // t.t.Transactions ([]actors.MTransaction) + // t.t.Transactions ([]actors.MTransaction) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Transactions)))); err != nil { return err } @@ -1021,7 +1083,7 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Signers ([]address.Address) + // t.t.Signers ([]address.Address) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1047,7 +1109,7 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { t.Signers[i] = v } - // t.t.Required (uint64) + // t.t.Required (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1056,8 +1118,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Required = extra - // t.t.NextTxID (uint64) + t.Required = uint64(extra) + // t.t.NextTxID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1066,8 +1128,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.NextTxID = extra - // t.t.InitialBalance (types.BigInt) + t.NextTxID = uint64(extra) + // t.t.InitialBalance (types.BigInt) (struct) { @@ -1076,7 +1138,7 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.StartingBlock (uint64) + // t.t.StartingBlock (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1085,8 +1147,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.StartingBlock = extra - // t.t.UnlockDuration (uint64) + t.StartingBlock = uint64(extra) + // t.t.UnlockDuration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1095,8 +1157,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.UnlockDuration = extra - // t.t.Transactions ([]actors.MTransaction) + t.UnlockDuration = uint64(extra) + // t.t.Transactions ([]actors.MTransaction) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1134,7 +1196,7 @@ func (t *MultiSigConstructorParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Signers ([]address.Address) + // t.t.Signers ([]address.Address) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Signers)))); err != nil { return err } @@ -1144,13 +1206,13 @@ func (t *MultiSigConstructorParams) MarshalCBOR(w io.Writer) error { } } - // t.t.Required (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Required)); err != nil { + // t.t.Required (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Required))); err != nil { return err } - // t.t.UnlockDuration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.UnlockDuration)); err != nil { + // t.t.UnlockDuration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.UnlockDuration))); err != nil { return err } return nil @@ -1171,7 +1233,7 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Signers ([]address.Address) + // t.t.Signers ([]address.Address) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1197,7 +1259,7 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { t.Signers[i] = v } - // t.t.Required (uint64) + // t.t.Required (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1206,8 +1268,8 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Required = extra - // t.t.UnlockDuration (uint64) + t.Required = uint64(extra) + // t.t.UnlockDuration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1216,7 +1278,7 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.UnlockDuration = extra + t.UnlockDuration = uint64(extra) return nil } @@ -1229,22 +1291,22 @@ func (t *MultiSigProposeParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) if err := t.Value.MarshalCBOR(w); err != nil { return err } - // t.t.Method (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Method)); err != nil { + // t.t.Method (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { return err } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Params)))); err != nil { return err } @@ -1269,7 +1331,7 @@ func (t *MultiSigProposeParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -1278,7 +1340,7 @@ func (t *MultiSigProposeParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) { @@ -1287,7 +1349,7 @@ func (t *MultiSigProposeParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Method (uint64) + // t.t.Method (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1296,8 +1358,8 @@ func (t *MultiSigProposeParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Method = extra - // t.t.Params ([]uint8) + t.Method = uint64(extra) + // t.t.Params ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1326,8 +1388,8 @@ func (t *MultiSigTxID) MarshalCBOR(w io.Writer) error { return err } - // t.t.TxID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.TxID)); err != nil { + // t.t.TxID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TxID))); err != nil { return err } return nil @@ -1348,7 +1410,7 @@ func (t *MultiSigTxID) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.TxID (uint64) + // t.t.TxID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1357,7 +1419,7 @@ func (t *MultiSigTxID) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.TxID = extra + t.TxID = uint64(extra) return nil } @@ -1370,12 +1432,12 @@ func (t *MultiSigSwapSignerParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) if err := t.From.MarshalCBOR(w); err != nil { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } @@ -1397,7 +1459,7 @@ func (t *MultiSigSwapSignerParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) { @@ -1406,7 +1468,7 @@ func (t *MultiSigSwapSignerParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -1427,8 +1489,8 @@ func (t *MultiSigChangeReqParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Req (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Req)); err != nil { + // t.t.Req (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Req))); err != nil { return err } return nil @@ -1449,7 +1511,7 @@ func (t *MultiSigChangeReqParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Req (uint64) + // t.t.Req (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1458,7 +1520,7 @@ func (t *MultiSigChangeReqParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Req = extra + t.Req = uint64(extra) return nil } @@ -1471,32 +1533,32 @@ func (t *MTransaction) MarshalCBOR(w io.Writer) error { return err } - // t.t.Created (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Created)); err != nil { + // t.t.Created (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Created))); err != nil { return err } - // t.t.TxID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.TxID)); err != nil { + // t.t.TxID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TxID))); err != nil { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) if err := t.Value.MarshalCBOR(w); err != nil { return err } - // t.t.Method (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Method)); err != nil { + // t.t.Method (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { return err } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Params)))); err != nil { return err } @@ -1504,7 +1566,7 @@ func (t *MTransaction) MarshalCBOR(w io.Writer) error { return err } - // t.t.Approved ([]address.Address) + // t.t.Approved ([]address.Address) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Approved)))); err != nil { return err } @@ -1514,18 +1576,18 @@ func (t *MTransaction) MarshalCBOR(w io.Writer) error { } } - // t.t.Complete (bool) + // t.t.Complete (bool) (bool) if err := cbg.WriteBool(w, t.Complete); err != nil { return err } - // t.t.Canceled (bool) + // t.t.Canceled (bool) (bool) if err := cbg.WriteBool(w, t.Canceled); err != nil { return err } - // t.t.RetCode (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.RetCode)); err != nil { + // t.t.RetCode (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.RetCode))); err != nil { return err } return nil @@ -1546,7 +1608,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Created (uint64) + // t.t.Created (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1555,8 +1617,8 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Created = extra - // t.t.TxID (uint64) + t.Created = uint64(extra) + // t.t.TxID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1565,8 +1627,8 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.TxID = extra - // t.t.To (address.Address) + t.TxID = uint64(extra) + // t.t.To (address.Address) (struct) { @@ -1575,7 +1637,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) { @@ -1584,7 +1646,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Method (uint64) + // t.t.Method (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1593,8 +1655,8 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Method = extra - // t.t.Params ([]uint8) + t.Method = uint64(extra) + // t.t.Params ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1611,7 +1673,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Params); err != nil { return err } - // t.t.Approved ([]address.Address) + // t.t.Approved ([]address.Address) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1637,7 +1699,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { t.Approved[i] = v } - // t.t.Complete (bool) + // t.t.Complete (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1654,7 +1716,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { default: return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) } - // t.t.Canceled (bool) + // t.t.Canceled (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1671,7 +1733,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { default: return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) } - // t.t.RetCode (uint64) + // t.t.RetCode (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1680,7 +1742,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.RetCode = extra + t.RetCode = uint64(extra) return nil } @@ -1693,12 +1755,12 @@ func (t *MultiSigRemoveSignerParam) MarshalCBOR(w io.Writer) error { return err } - // t.t.Signer (address.Address) + // t.t.Signer (address.Address) (struct) if err := t.Signer.MarshalCBOR(w); err != nil { return err } - // t.t.Decrease (bool) + // t.t.Decrease (bool) (bool) if err := cbg.WriteBool(w, t.Decrease); err != nil { return err } @@ -1720,7 +1782,7 @@ func (t *MultiSigRemoveSignerParam) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Signer (address.Address) + // t.t.Signer (address.Address) (struct) { @@ -1729,7 +1791,7 @@ func (t *MultiSigRemoveSignerParam) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Decrease (bool) + // t.t.Decrease (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1758,12 +1820,12 @@ func (t *MultiSigAddSignerParam) MarshalCBOR(w io.Writer) error { return err } - // t.t.Signer (address.Address) + // t.t.Signer (address.Address) (struct) if err := t.Signer.MarshalCBOR(w); err != nil { return err } - // t.t.Increase (bool) + // t.t.Increase (bool) (bool) if err := cbg.WriteBool(w, t.Increase); err != nil { return err } @@ -1785,7 +1847,7 @@ func (t *MultiSigAddSignerParam) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Signer (address.Address) + // t.t.Signer (address.Address) (struct) { @@ -1794,7 +1856,7 @@ func (t *MultiSigAddSignerParam) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Increase (bool) + // t.t.Increase (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1823,49 +1885,57 @@ func (t *PaymentChannelActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) if err := t.From.MarshalCBOR(w); err != nil { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } - // t.t.ToSend (types.BigInt) + // t.t.ToSend (types.BigInt) (struct) if err := t.ToSend.MarshalCBOR(w); err != nil { return err } - // t.t.ClosingAt (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ClosingAt)); err != nil { + // t.t.ClosingAt (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ClosingAt))); err != nil { return err } - // t.t.MinCloseHeight (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.MinCloseHeight)); err != nil { + // t.t.MinCloseHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinCloseHeight))); err != nil { return err } - // t.t.LaneStates (map[string]*actors.LaneState) - if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.LaneStates))); err != nil { - return err - } - - for k, v := range t.LaneStates { - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { - return err - } - if _, err := w.Write([]byte(k)); err != nil { + // t.t.LaneStates (map[string]*actors.LaneState) (map) + { + if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.LaneStates))); err != nil { return err } - if err := v.MarshalCBOR(w); err != nil { - return err + keys := make([]string, 0, len(t.LaneStates)) + for k := range t.LaneStates { + keys = append(keys, k) } + sort.Strings(keys) + for _, k := range keys { + v := t.LaneStates[k] + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { + return err + } + if _, err := w.Write([]byte(k)); err != nil { + return err + } + + if err := v.MarshalCBOR(w); err != nil { + return err + } + + } } return nil } @@ -1885,7 +1955,7 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) { @@ -1894,7 +1964,7 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -1903,7 +1973,7 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ToSend (types.BigInt) + // t.t.ToSend (types.BigInt) (struct) { @@ -1912,7 +1982,7 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ClosingAt (uint64) + // t.t.ClosingAt (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1921,8 +1991,8 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ClosingAt = extra - // t.t.MinCloseHeight (uint64) + t.ClosingAt = uint64(extra) + // t.t.MinCloseHeight (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1931,8 +2001,8 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.MinCloseHeight = extra - // t.t.LaneStates (map[string]*actors.LaneState) + t.MinCloseHeight = uint64(extra) + // t.t.LaneStates (map[string]*actors.LaneState) (map) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1997,7 +2067,7 @@ func (t *PCAConstructorParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } @@ -2019,7 +2089,7 @@ func (t *PCAConstructorParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -2040,18 +2110,18 @@ func (t *LaneState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Closed (bool) + // t.t.Closed (bool) (bool) if err := cbg.WriteBool(w, t.Closed); err != nil { return err } - // t.t.Redeemed (types.BigInt) + // t.t.Redeemed (types.BigInt) (struct) if err := t.Redeemed.MarshalCBOR(w); err != nil { return err } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } return nil @@ -2072,7 +2142,7 @@ func (t *LaneState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Closed (bool) + // t.t.Closed (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2089,7 +2159,7 @@ func (t *LaneState) UnmarshalCBOR(r io.Reader) error { default: return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) } - // t.t.Redeemed (types.BigInt) + // t.t.Redeemed (types.BigInt) (struct) { @@ -2098,7 +2168,7 @@ func (t *LaneState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Nonce (uint64) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2107,7 +2177,7 @@ func (t *LaneState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra + t.Nonce = uint64(extra) return nil } @@ -2120,12 +2190,12 @@ func (t *PCAUpdateChannelStateParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Sv (types.SignedVoucher) + // t.t.Sv (types.SignedVoucher) (struct) if err := t.Sv.MarshalCBOR(w); err != nil { return err } - // t.t.Secret ([]uint8) + // t.t.Secret ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Secret)))); err != nil { return err } @@ -2133,7 +2203,7 @@ func (t *PCAUpdateChannelStateParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -2158,7 +2228,7 @@ func (t *PCAUpdateChannelStateParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Sv (types.SignedVoucher) + // t.t.Sv (types.SignedVoucher) (struct) { @@ -2167,7 +2237,7 @@ func (t *PCAUpdateChannelStateParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Secret ([]uint8) + // t.t.Secret ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2184,7 +2254,7 @@ func (t *PCAUpdateChannelStateParams) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Secret); err != nil { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2213,17 +2283,17 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.PayChActor (address.Address) + // t.t.PayChActor (address.Address) (struct) if err := t.PayChActor.MarshalCBOR(w); err != nil { return err } - // t.t.Payer (address.Address) + // t.t.Payer (address.Address) (struct) if err := t.Payer.MarshalCBOR(w); err != nil { return err } - // t.t.ChannelMessage (cid.Cid) + // t.t.ChannelMessage (cid.Cid) (struct) if t.ChannelMessage == nil { if _, err := w.Write(cbg.CborNull); err != nil { @@ -2235,7 +2305,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { } } - // t.t.Vouchers ([]*types.SignedVoucher) + // t.t.Vouchers ([]*types.SignedVoucher) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil { return err } @@ -2262,7 +2332,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.PayChActor (address.Address) + // t.t.PayChActor (address.Address) (struct) { @@ -2271,7 +2341,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Payer (address.Address) + // t.t.Payer (address.Address) (struct) { @@ -2280,7 +2350,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ChannelMessage (cid.Cid) + // t.t.ChannelMessage (cid.Cid) (struct) { @@ -2304,7 +2374,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Vouchers ([]*types.SignedVoucher) + // t.t.Vouchers ([]*types.SignedVoucher) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2342,18 +2412,18 @@ func (t *StoragePowerState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Miners (cid.Cid) + // t.t.Miners (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Miners); err != nil { return xerrors.Errorf("failed to write cid field t.Miners: %w", err) } - // t.t.MinerCount (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.MinerCount)); err != nil { + // t.t.MinerCount (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinerCount))); err != nil { return err } - // t.t.TotalStorage (types.BigInt) + // t.t.TotalStorage (types.BigInt) (struct) if err := t.TotalStorage.MarshalCBOR(w); err != nil { return err } @@ -2375,7 +2445,7 @@ func (t *StoragePowerState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Miners (cid.Cid) + // t.t.Miners (cid.Cid) (struct) { @@ -2387,7 +2457,7 @@ func (t *StoragePowerState) UnmarshalCBOR(r io.Reader) error { t.Miners = c } - // t.t.MinerCount (uint64) + // t.t.MinerCount (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2396,8 +2466,8 @@ func (t *StoragePowerState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.MinerCount = extra - // t.t.TotalStorage (types.BigInt) + t.MinerCount = uint64(extra) + // t.t.TotalStorage (types.BigInt) (struct) { @@ -2418,22 +2488,22 @@ func (t *CreateStorageMinerParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) if err := t.Owner.MarshalCBOR(w); err != nil { return err } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) if err := t.Worker.MarshalCBOR(w); err != nil { return err } - // t.t.SectorSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { + // t.t.SectorSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorSize))); err != nil { return err } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.PeerID)))); err != nil { return err } @@ -2458,7 +2528,7 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) { @@ -2467,7 +2537,7 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) { @@ -2476,7 +2546,7 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.SectorSize (uint64) + // t.t.SectorSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2485,8 +2555,8 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorSize = extra - // t.t.PeerID (peer.ID) + t.SectorSize = uint64(extra) + // t.t.PeerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -2508,7 +2578,7 @@ func (t *IsMinerParam) MarshalCBOR(w io.Writer) error { return err } - // t.t.Addr (address.Address) + // t.t.Addr (address.Address) (struct) if err := t.Addr.MarshalCBOR(w); err != nil { return err } @@ -2530,7 +2600,7 @@ func (t *IsMinerParam) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Addr (address.Address) + // t.t.Addr (address.Address) (struct) { @@ -2551,7 +2621,7 @@ func (t *PowerLookupParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) if err := t.Miner.MarshalCBOR(w); err != nil { return err } @@ -2573,7 +2643,7 @@ func (t *PowerLookupParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) { @@ -2594,7 +2664,7 @@ func (t *UpdateStorageParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Delta (types.BigInt) + // t.t.Delta (types.BigInt) (struct) if err := t.Delta.MarshalCBOR(w); err != nil { return err } @@ -2616,7 +2686,7 @@ func (t *UpdateStorageParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Delta (types.BigInt) + // t.t.Delta (types.BigInt) (struct) { @@ -2637,12 +2707,12 @@ func (t *ArbitrateConsensusFaultParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Block1 (types.BlockHeader) + // t.t.Block1 (types.BlockHeader) (struct) if err := t.Block1.MarshalCBOR(w); err != nil { return err } - // t.t.Block2 (types.BlockHeader) + // t.t.Block2 (types.BlockHeader) (struct) if err := t.Block2.MarshalCBOR(w); err != nil { return err } @@ -2664,7 +2734,7 @@ func (t *ArbitrateConsensusFaultParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Block1 (types.BlockHeader) + // t.t.Block1 (types.BlockHeader) (struct) { @@ -2685,7 +2755,7 @@ func (t *ArbitrateConsensusFaultParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Block2 (types.BlockHeader) + // t.t.Block2 (types.BlockHeader) (struct) { @@ -2718,7 +2788,7 @@ func (t *PledgeCollateralParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Size (types.BigInt) + // t.t.Size (types.BigInt) (struct) if err := t.Size.MarshalCBOR(w); err != nil { return err } @@ -2740,7 +2810,7 @@ func (t *PledgeCollateralParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Size (types.BigInt) + // t.t.Size (types.BigInt) (struct) { @@ -2761,17 +2831,17 @@ func (t *MinerSlashConsensusFault) MarshalCBOR(w io.Writer) error { return err } - // t.t.Slasher (address.Address) + // t.t.Slasher (address.Address) (struct) if err := t.Slasher.MarshalCBOR(w); err != nil { return err } - // t.t.AtHeight (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.AtHeight)); err != nil { + // t.t.AtHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.AtHeight))); err != nil { return err } - // t.t.SlashedCollateral (types.BigInt) + // t.t.SlashedCollateral (types.BigInt) (struct) if err := t.SlashedCollateral.MarshalCBOR(w); err != nil { return err } @@ -2793,7 +2863,7 @@ func (t *MinerSlashConsensusFault) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Slasher (address.Address) + // t.t.Slasher (address.Address) (struct) { @@ -2802,7 +2872,7 @@ func (t *MinerSlashConsensusFault) UnmarshalCBOR(r io.Reader) error { } } - // t.t.AtHeight (uint64) + // t.t.AtHeight (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2811,8 +2881,8 @@ func (t *MinerSlashConsensusFault) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.AtHeight = extra - // t.t.SlashedCollateral (types.BigInt) + t.AtHeight = uint64(extra) + // t.t.SlashedCollateral (types.BigInt) (struct) { @@ -2833,12 +2903,12 @@ func (t *StorageParticipantBalance) MarshalCBOR(w io.Writer) error { return err } - // t.t.Locked (types.BigInt) + // t.t.Locked (types.BigInt) (struct) if err := t.Locked.MarshalCBOR(w); err != nil { return err } - // t.t.Available (types.BigInt) + // t.t.Available (types.BigInt) (struct) if err := t.Available.MarshalCBOR(w); err != nil { return err } @@ -2860,7 +2930,7 @@ func (t *StorageParticipantBalance) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Locked (types.BigInt) + // t.t.Locked (types.BigInt) (struct) { @@ -2869,7 +2939,7 @@ func (t *StorageParticipantBalance) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Available (types.BigInt) + // t.t.Available (types.BigInt) (struct) { @@ -2890,20 +2960,20 @@ func (t *StorageMarketState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Balances (cid.Cid) + // t.t.Balances (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Balances); err != nil { return xerrors.Errorf("failed to write cid field t.Balances: %w", err) } - // t.t.Deals (cid.Cid) + // t.t.Deals (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Deals); err != nil { return xerrors.Errorf("failed to write cid field t.Deals: %w", err) } - // t.t.NextDealID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.NextDealID)); err != nil { + // t.t.NextDealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextDealID))); err != nil { return err } return nil @@ -2924,7 +2994,7 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Balances (cid.Cid) + // t.t.Balances (cid.Cid) (struct) { @@ -2936,7 +3006,7 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { t.Balances = c } - // t.t.Deals (cid.Cid) + // t.t.Deals (cid.Cid) (struct) { @@ -2948,7 +3018,7 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { t.Deals = c } - // t.t.NextDealID (uint64) + // t.t.NextDealID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2957,7 +3027,7 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.NextDealID = extra + t.NextDealID = uint64(extra) return nil } @@ -2970,7 +3040,7 @@ func (t *WithdrawBalanceParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Balance (types.BigInt) + // t.t.Balance (types.BigInt) (struct) if err := t.Balance.MarshalCBOR(w); err != nil { return err } @@ -2992,7 +3062,7 @@ func (t *WithdrawBalanceParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Balance (types.BigInt) + // t.t.Balance (types.BigInt) (struct) { @@ -3013,7 +3083,7 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.PieceRef ([]uint8) + // t.t.PieceRef ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceRef)))); err != nil { return err } @@ -3021,47 +3091,47 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.PieceSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.PieceSize)); err != nil { + // t.t.PieceSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSize))); err != nil { return err } - // t.t.PieceSerialization (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.PieceSerialization)); err != nil { + // t.t.PieceSerialization (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSerialization))); err != nil { return err } - // t.t.Client (address.Address) + // t.t.Client (address.Address) (struct) if err := t.Client.MarshalCBOR(w); err != nil { return err } - // t.t.Provider (address.Address) + // t.t.Provider (address.Address) (struct) if err := t.Provider.MarshalCBOR(w); err != nil { return err } - // t.t.ProposalExpiration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ProposalExpiration)); err != nil { + // t.t.ProposalExpiration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { return err } - // t.t.Duration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Duration)); err != nil { + // t.t.Duration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { return err } - // t.t.StoragePricePerEpoch (types.BigInt) + // t.t.StoragePricePerEpoch (types.BigInt) (struct) if err := t.StoragePricePerEpoch.MarshalCBOR(w); err != nil { return err } - // t.t.StorageCollateral (types.BigInt) + // t.t.StorageCollateral (types.BigInt) (struct) if err := t.StorageCollateral.MarshalCBOR(w); err != nil { return err } - // t.t.ProposerSignature (types.Signature) + // t.t.ProposerSignature (types.Signature) (struct) if err := t.ProposerSignature.MarshalCBOR(w); err != nil { return err } @@ -3083,7 +3153,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.PieceRef ([]uint8) + // t.t.PieceRef ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3100,7 +3170,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.PieceRef); err != nil { return err } - // t.t.PieceSize (uint64) + // t.t.PieceSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3109,8 +3179,8 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.PieceSize = extra - // t.t.PieceSerialization (uint64) + t.PieceSize = uint64(extra) + // t.t.PieceSerialization (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3119,8 +3189,8 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.PieceSerialization = extra - // t.t.Client (address.Address) + t.PieceSerialization = uint64(extra) + // t.t.Client (address.Address) (struct) { @@ -3129,7 +3199,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Provider (address.Address) + // t.t.Provider (address.Address) (struct) { @@ -3138,7 +3208,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProposalExpiration (uint64) + // t.t.ProposalExpiration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3147,8 +3217,8 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ProposalExpiration = extra - // t.t.Duration (uint64) + t.ProposalExpiration = uint64(extra) + // t.t.Duration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3157,8 +3227,8 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Duration = extra - // t.t.StoragePricePerEpoch (types.BigInt) + t.Duration = uint64(extra) + // t.t.StoragePricePerEpoch (types.BigInt) (struct) { @@ -3167,7 +3237,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.StorageCollateral (types.BigInt) + // t.t.StorageCollateral (types.BigInt) (struct) { @@ -3176,7 +3246,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProposerSignature (types.Signature) + // t.t.ProposerSignature (types.Signature) (struct) { @@ -3209,12 +3279,12 @@ func (t *StorageDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) if err := t.Proposal.MarshalCBOR(w); err != nil { return err } - // t.t.CounterSignature (types.Signature) + // t.t.CounterSignature (types.Signature) (struct) if err := t.CounterSignature.MarshalCBOR(w); err != nil { return err } @@ -3236,7 +3306,7 @@ func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) { @@ -3245,7 +3315,7 @@ func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.CounterSignature (types.Signature) + // t.t.CounterSignature (types.Signature) (struct) { @@ -3278,7 +3348,7 @@ func (t *PublishStorageDealsParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Deals ([]actors.StorageDeal) + // t.t.Deals ([]actors.StorageDeal) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Deals)))); err != nil { return err } @@ -3305,7 +3375,7 @@ func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Deals ([]actors.StorageDeal) + // t.t.Deals ([]actors.StorageDeal) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3343,7 +3413,7 @@ func (t *PublishStorageDealResponse) MarshalCBOR(w io.Writer) error { return err } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { return err } @@ -3370,7 +3440,7 @@ func (t *PublishStorageDealResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3412,7 +3482,7 @@ func (t *ActivateStorageDealsParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Deals ([]uint64) + // t.t.Deals ([]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Deals)))); err != nil { return err } @@ -3439,7 +3509,7 @@ func (t *ActivateStorageDealsParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Deals ([]uint64) + // t.t.Deals ([]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3481,7 +3551,7 @@ func (t *ProcessStorageDealsPaymentParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { return err } @@ -3508,7 +3578,7 @@ func (t *ProcessStorageDealsPaymentParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3550,13 +3620,13 @@ func (t *OnChainDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.Deal (actors.StorageDeal) + // t.t.Deal (actors.StorageDeal) (struct) if err := t.Deal.MarshalCBOR(w); err != nil { return err } - // t.t.ActivationEpoch (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ActivationEpoch)); err != nil { + // t.t.ActivationEpoch (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ActivationEpoch))); err != nil { return err } return nil @@ -3577,7 +3647,7 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Deal (actors.StorageDeal) + // t.t.Deal (actors.StorageDeal) (struct) { @@ -3586,7 +3656,7 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ActivationEpoch (uint64) + // t.t.ActivationEpoch (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3595,6 +3665,199 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ActivationEpoch = extra + t.ActivationEpoch = uint64(extra) + return nil +} + +func (t *ComputeDataCommitmentParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.DealIDs ([]uint64) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + + // t.t.SectorSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorSize))); err != nil { + return err + } + return nil +} + +func (t *ComputeDataCommitmentParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.DealIDs ([]uint64) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + // t.t.SectorSize (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorSize = uint64(extra) + return nil +} + +func (t *SectorProveCommitInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Proof ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { + return err + } + if _, err := w.Write(t.Proof); err != nil { + return err + } + + // t.t.SectorID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { + return err + } + + // t.t.DealIDs ([]uint64) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + return nil +} + +func (t *SectorProveCommitInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Proof ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Proof: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Proof = make([]byte, extra) + if _, err := io.ReadFull(br, t.Proof); err != nil { + return err + } + // t.t.SectorID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = uint64(extra) + // t.t.DealIDs ([]uint64) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + return nil } diff --git a/chain/blocksync.go b/chain/blocksync.go index 8a9e629eb..71140f2e0 100644 --- a/chain/blocksync.go +++ b/chain/blocksync.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/node/modules/dtypes" blocks "github.com/ipfs/go-block-format" @@ -93,7 +93,7 @@ func (bss *BlockSyncService) HandleStream(s inet.Stream) { defer s.Close() var req BlockSyncRequest - if err := cborrpc.ReadCborRPC(bufio.NewReader(s), &req); err != nil { + if err := cborutil.ReadCborRPC(bufio.NewReader(s), &req); err != nil { log.Errorf("failed to read block sync request: %s", err) return } @@ -105,7 +105,7 @@ func (bss *BlockSyncService) HandleStream(s inet.Stream) { return } - if err := cborrpc.WriteCborRPC(s, resp); err != nil { + if err := cborutil.WriteCborRPC(s, resp); err != nil { log.Error("failed to write back response for handle stream: ", err) return } @@ -401,12 +401,12 @@ func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *Bloc return nil, err } - if err := cborrpc.WriteCborRPC(s, req); err != nil { + if err := cborutil.WriteCborRPC(s, req); err != nil { return nil, err } var res BlockSyncResponse - if err := cborrpc.ReadCborRPC(bufio.NewReader(s), &res); err != nil { + if err := cborutil.ReadCborRPC(bufio.NewReader(s), &res); err != nil { return nil, err } diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 40413170c..f5bb90d1a 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -24,7 +24,7 @@ func (t *AskRequest) MarshalCBOR(w io.Writer) error { return err } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) if err := t.Miner.MarshalCBOR(w); err != nil { return err } @@ -46,7 +46,7 @@ func (t *AskRequest) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) { @@ -67,7 +67,7 @@ func (t *AskResponse) MarshalCBOR(w io.Writer) error { return err } - // t.t.Ask (types.SignedStorageAsk) + // t.t.Ask (types.SignedStorageAsk) (struct) if err := t.Ask.MarshalCBOR(w); err != nil { return err } @@ -89,7 +89,7 @@ func (t *AskResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Ask (types.SignedStorageAsk) + // t.t.Ask (types.SignedStorageAsk) (struct) { @@ -118,14 +118,21 @@ func (t *Proposal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{129}); err != nil { + if _, err := w.Write([]byte{130}); err != nil { return err } - // t.t.DealProposal (actors.StorageDealProposal) + // t.t.DealProposal (actors.StorageDealProposal) (struct) if err := t.DealProposal.MarshalCBOR(w); err != nil { return err } + + // t.t.Piece (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Piece); err != nil { + return xerrors.Errorf("failed to write cid field t.Piece: %w", err) + } + return nil } @@ -140,17 +147,41 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 1 { + if extra != 2 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.DealProposal (actors.StorageDealProposal) + // t.t.DealProposal (actors.StorageDealProposal) (struct) { - if err := t.DealProposal.UnmarshalCBOR(br); err != nil { + pb, err := br.PeekByte() + if err != nil { return err } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.DealProposal = new(actors.StorageDealProposal) + if err := t.DealProposal.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + // t.t.Piece (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Piece: %w", err) + } + + t.Piece = c } return nil @@ -161,16 +192,16 @@ func (t *Response) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{134}); err != nil { + if _, err := w.Write([]byte{133}); err != nil { return err } - // t.t.State (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + // t.t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { return err } - // t.t.Message (string) + // t.t.Message (string) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { return err } @@ -178,18 +209,18 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proposal (cid.Cid) + // t.t.Proposal (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Proposal); err != nil { return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) } - // t.t.StorageDeal (actors.StorageDeal) + // t.t.StorageDeal (actors.StorageDeal) (struct) if err := t.StorageDeal.MarshalCBOR(w); err != nil { return err } - // t.t.PublishMessage (cid.Cid) + // t.t.PublishMessage (cid.Cid) (struct) if t.PublishMessage == nil { if _, err := w.Write(cbg.CborNull); err != nil { @@ -201,18 +232,6 @@ func (t *Response) MarshalCBOR(w io.Writer) error { } } - // t.t.CommitMessage (cid.Cid) - - if t.CommitMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.CommitMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err) - } - } - return nil } @@ -227,11 +246,11 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 6 { + if extra != 5 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.State (uint64) + // t.t.State (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -240,8 +259,8 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.State = extra - // t.t.Message (string) + t.State = uint64(extra) + // t.t.Message (string) (string) { sval, err := cbg.ReadString(br) @@ -251,7 +270,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.Message = string(sval) } - // t.t.Proposal (cid.Cid) + // t.t.Proposal (cid.Cid) (struct) { @@ -263,7 +282,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.Proposal = c } - // t.t.StorageDeal (actors.StorageDeal) + // t.t.StorageDeal (actors.StorageDeal) (struct) { @@ -284,7 +303,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { } } - // t.t.PublishMessage (cid.Cid) + // t.t.PublishMessage (cid.Cid) (struct) { @@ -307,30 +326,6 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.PublishMessage = &c } - } - // t.t.CommitMessage (cid.Cid) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) - } - - t.CommitMessage = &c - } - } return nil } @@ -344,12 +339,12 @@ func (t *SignedResponse) MarshalCBOR(w io.Writer) error { return err } - // t.t.Response (deals.Response) + // t.t.Response (deals.Response) (struct) if err := t.Response.MarshalCBOR(w); err != nil { return err } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) if err := t.Signature.MarshalCBOR(w); err != nil { return err } @@ -371,7 +366,7 @@ func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Response (deals.Response) + // t.t.Response (deals.Response) (struct) { @@ -380,7 +375,7 @@ func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) { @@ -409,42 +404,47 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{135}); err != nil { + if _, err := w.Write([]byte{136}); err != nil { return err } - // t.t.Data (cid.Cid) + // t.t.Data (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Data); err != nil { return xerrors.Errorf("failed to write cid field t.Data: %w", err) } - // t.t.PricePerEpoch (types.BigInt) + // t.t.PricePerEpoch (types.BigInt) (struct) if err := t.PricePerEpoch.MarshalCBOR(w); err != nil { return err } - // t.t.ProposalExpiration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ProposalExpiration)); err != nil { + // t.t.ProposalExpiration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { return err } - // t.t.Duration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Duration)); err != nil { + // t.t.Duration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { return err } - // t.t.ProviderAddress (address.Address) + // t.t.ProviderAddress (address.Address) (struct) if err := t.ProviderAddress.MarshalCBOR(w); err != nil { return err } - // t.t.Client (address.Address) + // t.t.Client (address.Address) (struct) if err := t.Client.MarshalCBOR(w); err != nil { return err } - // t.t.MinerID (peer.ID) + // t.t.MinerWorker (address.Address) (struct) + if err := t.MinerWorker.MarshalCBOR(w); err != nil { + return err + } + + // t.t.MinerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil { return err } @@ -465,11 +465,11 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 7 { + if extra != 8 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Data (cid.Cid) + // t.t.Data (cid.Cid) (struct) { @@ -481,7 +481,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { t.Data = c } - // t.t.PricePerEpoch (types.BigInt) + // t.t.PricePerEpoch (types.BigInt) (struct) { @@ -490,7 +490,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProposalExpiration (uint64) + // t.t.ProposalExpiration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -499,8 +499,8 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ProposalExpiration = extra - // t.t.Duration (uint64) + t.ProposalExpiration = uint64(extra) + // t.t.Duration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -509,8 +509,8 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Duration = extra - // t.t.ProviderAddress (address.Address) + t.Duration = uint64(extra) + // t.t.ProviderAddress (address.Address) (struct) { @@ -519,7 +519,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Client (address.Address) + // t.t.Client (address.Address) (struct) { @@ -528,7 +528,16 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.MinerID (peer.ID) + // t.t.MinerWorker (address.Address) (struct) + + { + + if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.MinerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -546,33 +555,56 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{132}); err != nil { + if _, err := w.Write([]byte{135}); err != nil { return err } - // t.t.ProposalCid (cid.Cid) + // t.t.ProposalCid (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ProposalCid); err != nil { return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) if err := t.Proposal.MarshalCBOR(w); err != nil { return err } - // t.t.State (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + // t.t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { return err } - // t.t.Miner (peer.ID) + // t.t.Miner (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { return err } if _, err := w.Write([]byte(t.Miner)); err != nil { return err } + + // t.t.MinerWorker (address.Address) (struct) + if err := t.MinerWorker.MarshalCBOR(w); err != nil { + return err + } + + // t.t.DealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { + return err + } + + // t.t.PublishMessage (cid.Cid) (struct) + + if t.PublishMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) + } + } + return nil } @@ -587,11 +619,11 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 4 { + if extra != 7 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.ProposalCid (cid.Cid) + // t.t.ProposalCid (cid.Cid) (struct) { @@ -603,7 +635,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { t.ProposalCid = c } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) { @@ -612,7 +644,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.State (uint64) + // t.t.State (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -621,8 +653,8 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.State = extra - // t.t.Miner (peer.ID) + t.State = uint64(extra) + // t.t.Miner (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -632,6 +664,49 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { t.Miner = peer.ID(sval) } + // t.t.MinerWorker (address.Address) (struct) + + { + + if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.DealID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + // t.t.PublishMessage (cid.Cid) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) + } + + t.PublishMessage = &c + } + + } return nil } @@ -644,7 +719,7 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.Client (peer.ID) + // t.t.Client (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil { return err } @@ -652,35 +727,35 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) if err := t.Proposal.MarshalCBOR(w); err != nil { return err } - // t.t.ProposalCid (cid.Cid) + // t.t.ProposalCid (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ProposalCid); err != nil { return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) } - // t.t.State (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + // t.t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { return err } - // t.t.Ref (cid.Cid) + // t.t.Ref (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Ref); err != nil { return xerrors.Errorf("failed to write cid field t.Ref: %w", err) } - // t.t.DealID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.DealID)); err != nil { + // t.t.DealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { return err } - // t.t.SectorID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorID)); err != nil { + // t.t.SectorID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { return err } return nil @@ -701,7 +776,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Client (peer.ID) + // t.t.Client (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -711,7 +786,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { t.Client = peer.ID(sval) } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) { @@ -720,7 +795,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProposalCid (cid.Cid) + // t.t.ProposalCid (cid.Cid) (struct) { @@ -732,7 +807,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { t.ProposalCid = c } - // t.t.State (uint64) + // t.t.State (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -741,8 +816,8 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.State = extra - // t.t.Ref (cid.Cid) + t.State = uint64(extra) + // t.t.Ref (cid.Cid) (struct) { @@ -754,7 +829,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { t.Ref = c } - // t.t.DealID (uint64) + // t.t.DealID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -763,8 +838,8 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.DealID = extra - // t.t.SectorID (uint64) + t.DealID = uint64(extra) + // t.t.SectorID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -773,6 +848,6 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorID = extra + t.SectorID = uint64(extra) return nil } diff --git a/chain/deals/client.go b/chain/deals/client.go index fa70dbe99..eb96be154 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -2,7 +2,6 @@ package deals import ( "context" - "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -16,11 +15,15 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/events" + "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" + "github.com/filecoin-project/lotus/lib/statestore" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/retrieval/discovery" ) @@ -32,6 +35,10 @@ type ClientDeal struct { Proposal actors.StorageDealProposal State api.DealState Miner peer.ID + MinerWorker address.Address + DealID uint64 + + PublishMessage *cid.Cid s inet.Stream } @@ -43,9 +50,10 @@ type Client struct { w *wallet.Wallet dag dtypes.ClientDAG discovery *discovery.Local - mpool full.MpoolAPI + events *events.Events + fm *market.FundMgr - deals ClientStateStore + deals *statestore.StateStore conns map[cid.Cid]inet.Stream incoming chan *ClientDeal @@ -59,9 +67,10 @@ type clientDealUpdate struct { newState api.DealState id cid.Cid err error + mut func(*ClientDeal) } -func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, mpool full.MpoolAPI) *Client { +func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, fm *market.FundMgr, chainapi full.ChainAPI) *Client { c := &Client{ sm: sm, chain: chain, @@ -69,9 +78,10 @@ func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w * w: w, dag: dag, discovery: discovery, - mpool: mpool, + fm: fm, + events: events.NewEvents(context.TODO(), &chainapi), - deals: ClientStateStore{StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}}, + deals: statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))), conns: map[cid.Cid]inet.Stream{}, incoming: make(chan *ClientDeal, 16), @@ -127,10 +137,13 @@ func (c *Client) onIncoming(deal *ClientDeal) { } func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { - log.Infof("Deal %s updated state to %d", update.id, update.newState) + log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState]) var deal ClientDeal - err := c.deals.MutateClient(update.id, func(d *ClientDeal) error { + err := c.deals.Mutate(update.id, func(d *ClientDeal) error { d.State = update.newState + if update.mut != nil { + update.mut(d) + } deal = *d return nil }) @@ -152,7 +165,8 @@ func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { case api.DealStaged: c.handle(ctx, deal, c.staged, api.DealSealing) case api.DealSealing: - c.handle(ctx, deal, c.sealing, api.DealComplete) + c.handle(ctx, deal, c.sealing, api.DealNoUpdate) + // TODO: DealComplete -> watch for faults, expiration, etc. } } @@ -165,80 +179,59 @@ type ClientDealProposal struct { ProviderAddress address.Address Client address.Address + MinerWorker address.Address MinerID peer.ID } func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - // check market funds - clientMarketBalance, err := c.sm.MarketBalance(ctx, p.Client, nil) - if err != nil { - return cid.Undef, xerrors.Errorf("getting client market balance failed: %w", err) + if err := c.fm.EnsureAvailable(ctx, p.Client, types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))); err != nil { + return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) } - if clientMarketBalance.Available.LessThan(types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))) { - // TODO: move to a smarter market funds manager + commP, pieceSize, err := c.commP(ctx, p.Data) - smsg, err := c.mpool.MpoolPushMessage(ctx, &types.Message{ - To: actors.StorageMarketAddress, - From: p.Client, - Value: types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration)), - GasPrice: types.NewInt(0), - GasLimit: types.NewInt(1000000), - Method: actors.SMAMethods.AddBalance, - }) - if err != nil { - return cid.Undef, err - } - - _, r, err := c.sm.WaitForMessage(ctx, smsg.Cid()) - if err != nil { - return cid.Undef, err - } - - if r.ExitCode != 0 { - return cid.Undef, xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.ExitCode) - } - } - - dataSize, err := c.dataSize(ctx, p.Data) - - proposal := &actors.StorageDealProposal{ - PieceRef: p.Data.Bytes(), - PieceSize: uint64(dataSize), + dealProposal := &actors.StorageDealProposal{ + PieceRef: commP, + PieceSize: uint64(pieceSize), PieceSerialization: actors.SerializationUnixFSv0, Client: p.Client, Provider: p.ProviderAddress, ProposalExpiration: p.ProposalExpiration, Duration: p.Duration, StoragePricePerEpoch: p.PricePerEpoch, - StorageCollateral: types.NewInt(uint64(dataSize)), // TODO: real calc + StorageCollateral: types.NewInt(uint64(pieceSize)), // TODO: real calc } - if err := api.SignWith(ctx, c.w.Sign, p.Client, proposal); err != nil { + if err := api.SignWith(ctx, c.w.Sign, p.Client, dealProposal); err != nil { return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) } - proposalNd, err := cborrpc.AsIpld(proposal) + proposalNd, err := cborutil.AsIpld(dealProposal) if err != nil { return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err) } s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID) if err != nil { - s.Reset() return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err) } - if err := cborrpc.WriteCborRPC(s, proposal); err != nil { + proposal := &Proposal{ + DealProposal: dealProposal, + Piece: p.Data, + } + + if err := cborutil.WriteCborRPC(s, proposal); err != nil { s.Reset() return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err) } deal := &ClientDeal{ ProposalCid: proposalNd.Cid(), - Proposal: *proposal, + Proposal: *dealProposal, State: api.DealUnknown, Miner: p.MinerID, + MinerWorker: p.MinerWorker, s: s, } @@ -246,7 +239,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro c.incoming <- deal return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{ - Address: proposal.Provider, + Address: dealProposal.Provider, ID: deal.Miner, }) } @@ -260,12 +253,12 @@ func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*t req := &AskRequest{ Miner: a, } - if err := cborrpc.WriteCborRPC(s, req); err != nil { + if err := cborutil.WriteCborRPC(s, req); err != nil { return nil, xerrors.Errorf("failed to send ask request: %w", err) } var out AskResponse - if err := cborrpc.ReadCborRPC(s, &out); err != nil { + if err := cborutil.ReadCborRPC(s, &out); err != nil { return nil, xerrors.Errorf("failed to read ask response: %w", err) } @@ -285,7 +278,19 @@ func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*t } func (c *Client) List() ([]ClientDeal, error) { - return c.deals.ListClient() + var out []ClientDeal + if err := c.deals.List(&out); err != nil { + return nil, err + } + return out, nil +} + +func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) { + var out ClientDeal + if err := c.deals.Get(d, &out); err != nil { + return nil, err + } + return &out, nil } func (c *Client) Stop() { diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index bb30e8440..c01d0f430 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -1,142 +1,225 @@ package deals import ( + "bytes" "context" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/cborutil" ) -type clientHandlerFunc func(ctx context.Context, deal ClientDeal) error +type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) { go func() { - err := cb(ctx, deal) + mut, err := cb(ctx, deal) if err != nil { next = api.DealError } + + if err == nil && next == api.DealNoUpdate { + return + } + select { case c.updated <- clientDealUpdate{ newState: next, id: deal.ProposalCid, err: err, + mut: mut, }: case <-c.stop: } }() } -func (c *Client) new(ctx context.Context, deal ClientDeal) error { +func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { resp, err := c.readStorageDealResp(deal) if err != nil { - return err + return nil, err + } + if err := c.disconnect(deal); err != nil { + return nil, err } + /* data transfer happens */ if resp.State != api.DealAccepted { - return xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State) + return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State) } - // TODO: spec says it's optional - pubmsg, err := c.chain.GetMessage(*resp.PublishMessage) + return func(info *ClientDeal) { + info.PublishMessage = resp.PublishMessage + }, nil +} + +func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { + log.Infow("DEAL ACCEPTED!") + + pubmsg, err := c.chain.GetMessage(*deal.PublishMessage) if err != nil { - return xerrors.Errorf("getting deal pubsish message: %w", err) + return nil, xerrors.Errorf("getting deal pubsish message: %w", err) } pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) if err != nil { - return xerrors.Errorf("getting miner worker failed: %w", err) + return nil, xerrors.Errorf("getting miner worker failed: %w", err) } if pubmsg.From != pw { - return xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) + return nil, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) } if pubmsg.To != actors.StorageMarketAddress { - return xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) + return nil, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) } if pubmsg.Method != actors.SMAMethods.PublishStorageDeals { - return xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) + return nil, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) + } + + var params actors.PublishStorageDealsParams + if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil { + return nil, err + } + + dealIdx := -1 + for i, storageDeal := range params.Deals { + // TODO: make it less hacky + eq, err := cborutil.Equals(&deal.Proposal, &storageDeal.Proposal) + if err != nil { + return nil, err + } + if eq { + dealIdx = i + break + } + } + + if dealIdx == -1 { + return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage) } // TODO: timeout - _, ret, err := c.sm.WaitForMessage(ctx, *resp.PublishMessage) + _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage) if err != nil { - return xerrors.Errorf("waiting for deal publish message: %w", err) + return nil, xerrors.Errorf("waiting for deal publish message: %w", err) } if ret.ExitCode != 0 { - return xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) + return nil, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) } - // TODO: persist dealId - log.Info("DEAL ACCEPTED!") + var res actors.PublishStorageDealResponse + if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { + return nil, err + } - return nil + return func(info *ClientDeal) { + info.DealID = res.DealIDs[dealIdx] + }, nil } -func (c *Client) accepted(ctx context.Context, deal ClientDeal) error { - /* data transfer happens */ +func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { + // TODO: Maybe wait for pre-commit - resp, err := c.readStorageDealResp(deal) - if err != nil { - return err - } - - if resp.State != api.DealStaged { - return xerrors.Errorf("deal wasn't staged (State=%d)", resp.State) - } - - log.Info("DEAL STAGED!") - - return nil + return nil, nil } -func (c *Client) staged(ctx context.Context, deal ClientDeal) error { - /* miner seals our data, hopefully */ +func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { + checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) + if err != nil { + // TODO: This may be fine for some errors + return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err) + } - resp, err := c.readStorageDealResp(deal) - if err != nil { - return err + if sd.ActivationEpoch > 0 { + select { + case c.updated <- clientDealUpdate{ + newState: api.DealComplete, + id: deal.ProposalCid, + }: + case <-c.stop: + } + + return true, false, nil + } + + return false, true, nil } - if resp.State != api.DealSealing { - return xerrors.Errorf("deal wasn't sealed (State=%d)", resp.State) + called := func(msg *types.Message, ts *types.TipSet, curH uint64) (more bool, err error) { + defer func() { + if err != nil { + select { + case c.updated <- clientDealUpdate{ + newState: api.DealComplete, + id: deal.ProposalCid, + err: xerrors.Errorf("handling applied event: %w", err), + }: + case <-c.stop: + } + } + }() + + if msg == nil { + log.Error("timed out waiting for deal activation... what now?") + return false, nil + } + + var params actors.SectorProveCommitInfo + if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { + return false, err + } + + var found bool + for _, dealID := range params.DealIDs { + if dealID == deal.DealID { + found = true + break + } + } + + if !found { + return true, nil + } + + sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) + if err != nil { + return false, xerrors.Errorf("failed to look up deal on chain: %w", err) + } + + if sd.ActivationEpoch == 0 { + return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", deal.DealID, ts.ParentState(), ts.Height()) + } + + log.Infof("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch) + + select { + case c.updated <- clientDealUpdate{ + newState: api.DealComplete, + id: deal.ProposalCid, + }: + case <-c.stop: + } + + return false, nil } - log.Info("DEAL SEALED!") - - // TODO: want? - /*ssize, err := stmgr.GetMinerSectorSize(ctx, c.sm, nil, deal.Proposal.MinerAddress) - if err != nil { - return xerrors.Errorf("failed to get miner sector size: %w", err) + revert := func(ctx context.Context, ts *types.TipSet) error { + log.Warn("deal activation reverted; TODO: actually handle this!") + // TODO: Just go back to DealSealing? + return nil } - ok, err := sectorbuilder.VerifyPieceInclusionProof(ssize, deal.Proposal.Size, deal.Proposal.CommP, resp.CommD, resp.PieceInclusionProof.ProofElements) - if err != nil { - return xerrors.Errorf("verifying piece inclusion proof in staged deal %s: %w", deal.ProposalCid, err) + if err := c.events.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, deal.Proposal.Provider, actors.MAMethods.ProveCommitSector); err != nil { + return nil, xerrors.Errorf("failed to set up called handler") } - if !ok { - return xerrors.Errorf("verifying piece inclusion proof in staged deal %s failed", deal.ProposalCid) - }*/ - return nil -} - -func (c *Client) sealing(ctx context.Context, deal ClientDeal) error { - resp, err := c.readStorageDealResp(deal) - if err != nil { - return err - } - - if resp.State != api.DealComplete { - return xerrors.Errorf("deal wasn't complete (State=%d)", resp.State) - } - - // TODO: look for the commit message on chain, negotiate better payment vouchers - - log.Info("DEAL COMPLETE!!") - return nil + return nil, nil } diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 9c26f4268..e4c11eb58 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -9,7 +9,9 @@ import ( unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" + "github.com/filecoin-project/lotus/lib/padreader" + "github.com/filecoin-project/lotus/lib/sectorbuilder" ) func (c *Client) failDeal(id cid.Cid, cerr error) { @@ -25,29 +27,41 @@ func (c *Client) failDeal(id cid.Cid, cerr error) { } // TODO: store in some sort of audit log - log.Errorf("deal %s failed: %s", id, cerr) + log.Errorf("deal %s failed: %+v", id, cerr) } -func (c *Client) dataSize(ctx context.Context, data cid.Cid) (int64, error) { +func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) { root, err := c.dag.Get(ctx, data) if err != nil { log.Errorf("failed to get file root for deal: %s", err) - return 0, err + return nil, 0, err } n, err := unixfile.NewUnixfsFile(ctx, c.dag, root) if err != nil { log.Errorf("cannot open unixfs file: %s", err) - return 0, err + return nil, 0, err } uf, ok := n.(files.File) if !ok { // TODO: we probably got directory, how should we handle this in unixfs mode? - return 0, xerrors.New("unsupported unixfs type") + return nil, 0, xerrors.New("unsupported unixfs type") } - return uf.Size() + s, err := uf.Size() + if err != nil { + return nil, 0, err + } + + pr, psize := padreader.New(uf, uint64(s)) + + commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize) + if err != nil { + return nil, 0, xerrors.Errorf("generating CommP: %w", err) + } + + return commp[:], psize, nil } func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { @@ -58,16 +72,29 @@ func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { } var resp SignedResponse - if err := cborrpc.ReadCborRPC(s, &resp); err != nil { + if err := cborutil.ReadCborRPC(s, &resp); err != nil { log.Errorw("failed to read Response message", "error", err) return nil, err } - // TODO: verify signature + if err := resp.Verify(deal.MinerWorker); err != nil { + return nil, xerrors.Errorf("verifying response signature failed", err) + } if resp.Response.Proposal != deal.ProposalCid { - return nil, xerrors.New("miner responded to a wrong proposal") + return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid) } return &resp.Response, nil } + +func (c *Client) disconnect(deal ClientDeal) error { + s, ok := c.conns[deal.ProposalCid] + if !ok { + return nil + } + + err := s.Close() + delete(c.conns, deal.ProposalCid) + return err +} diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 86383c61b..2c8978170 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -15,9 +15,10 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" + "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage/commitment" + "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -42,15 +43,15 @@ type Provider struct { ask *types.SignedStorageAsk askLk sync.Mutex - secst *sectorblocks.SectorBlocks - commt *commitment.Tracker - full api.FullNode + secb *sectorblocks.SectorBlocks + sminer *storage.Miner + full api.FullNode // TODO: Use a custom protocol or graphsync in the future // TODO: GC dag dtypes.StagingDAG - deals MinerStateStore + deals *statestore.StateStore ds dtypes.MetadataDS conns map[cid.Cid]inet.Stream @@ -70,7 +71,7 @@ type minerDealUpdate struct { mut func(*MinerDeal) } -func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt *commitment.Tracker, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) { +func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) { addr, err := ds.Get(datastore.NewKey("miner-address")) if err != nil { return nil, err @@ -81,13 +82,13 @@ func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt * } h := &Provider{ - secst: secst, - commt: commt, - dag: dag, - full: fullNode, + sminer: sminer, + dag: dag, + full: fullNode, + secb: secb, pricePerByteBlock: types.NewInt(3), // TODO: allow setting - minPieceSize: 1, + minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up)) conns: map[cid.Cid]inet.Stream{}, @@ -98,7 +99,7 @@ func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt * actor: minerAddress, - deals: MinerStateStore{StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}}, + deals: statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))), ds: ds, } @@ -159,14 +160,14 @@ func (p *Provider) onIncoming(deal MinerDeal) { } func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { - log.Infof("Deal %s updated state to %d", update.id, update.newState) + log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState]) if update.err != nil { - log.Errorf("deal %s (newSt: %d) failed: %s", update.id, update.newState, update.err) + log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err) p.failDeal(update.id, update.err) return } var deal MinerDeal - err := p.deals.MutateMiner(update.id, func(d *MinerDeal) error { + err := p.deals.Mutate(update.id, func(d *MinerDeal) error { d.State = update.newState if update.mut != nil { update.mut(d) @@ -191,24 +192,19 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { } } -func (p *Provider) newDeal(s inet.Stream, proposal actors.StorageDealProposal) (MinerDeal, error) { - proposalNd, err := cborrpc.AsIpld(&proposal) - if err != nil { - return MinerDeal{}, err - } - - ref, err := cid.Cast(proposal.PieceRef) +func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) { + proposalNd, err := cborutil.AsIpld(proposal.DealProposal) if err != nil { return MinerDeal{}, err } return MinerDeal{ Client: s.Conn().RemotePeer(), - Proposal: proposal, + Proposal: *proposal.DealProposal, ProposalCid: proposalNd.Cid(), State: api.DealUnknown, - Ref: ref, + Ref: proposal.Piece, s: s, }, nil @@ -226,7 +222,7 @@ func (p *Provider) HandleStream(s inet.Stream) { deal, err := p.newDeal(s, proposal) if err != nil { - log.Error(err) + log.Errorf("%+v", err) s.Close() return } diff --git a/chain/deals/provider_asks.go b/chain/deals/provider_asks.go index 1b1960b97..1fc8e69ed 100644 --- a/chain/deals/provider_asks.go +++ b/chain/deals/provider_asks.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" datastore "github.com/ipfs/go-datastore" inet "github.com/libp2p/go-libp2p-core/network" "golang.org/x/xerrors" @@ -54,14 +54,14 @@ func (p *Provider) getAsk(m address.Address) *types.SignedStorageAsk { func (p *Provider) HandleAskStream(s inet.Stream) { defer s.Close() var ar AskRequest - if err := cborrpc.ReadCborRPC(s, &ar); err != nil { + if err := cborutil.ReadCborRPC(s, &ar); err != nil { log.Errorf("failed to read AskRequest from incoming stream: %s", err) return } resp := p.processAskRequest(&ar) - if err := cborrpc.WriteCborRPC(s, resp); err != nil { + if err := cborutil.WriteCborRPC(s, resp); err != nil { log.Errorf("failed to write ask response: %s", err) return } @@ -98,7 +98,7 @@ func (p *Provider) loadAsk() error { } var ssa types.SignedStorageAsk - if err := cborrpc.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil { + if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil { return err } @@ -107,7 +107,7 @@ func (p *Provider) loadAsk() error { } func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) { - b, err := cborrpc.Dump(a) + b, err := cborutil.Dump(a) if err != nil { return nil, err } @@ -129,7 +129,7 @@ func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) } func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { - b, err := cborrpc.Dump(a) + b, err := cborutil.Dump(a) if err != nil { return err } @@ -150,7 +150,7 @@ func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { return xerrors.Errorf("failed to get worker for miner in ask", err) } - sigb, err := cborrpc.Dump(ask.Ask) + sigb, err := cborutil.Dump(ask.Ask) if err != nil { return xerrors.Errorf("failed to re-serialize ask") } diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 595f236da..35f339502 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -4,17 +4,14 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-sectorbuilder/sealing_state" - "github.com/ipfs/go-cid" "github.com/ipfs/go-merkledag" unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -41,33 +38,6 @@ func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandle } // ACCEPTED - -func (p *Provider) addMarketFunds(ctx context.Context, worker address.Address, deal MinerDeal) error { - log.Info("Adding market funds for storage collateral") - smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{ - To: actors.StorageMarketAddress, - From: worker, - Value: deal.Proposal.StorageCollateral, - GasPrice: types.NewInt(0), - GasLimit: types.NewInt(1000000), - Method: actors.SMAMethods.AddBalance, - }) - if err != nil { - return err - } - - r, err := p.full.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - return err - } - - if r.Receipt.ExitCode != 0 { - return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode) - } - - return nil -} - func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { switch deal.Proposal.PieceSerialization { //case SerializationRaw: @@ -87,7 +57,6 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // TODO: check StorageCollateral - // TODO: minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30)) if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) { return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice) @@ -114,16 +83,9 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, err } - providerMarketBalance, err := p.full.StateMarketBalance(ctx, waddr, nil) - if err != nil { - return nil, xerrors.Errorf("getting provider market balance failed: %w", err) - } - - // TODO: this needs to be atomic - if providerMarketBalance.Available.LessThan(deal.Proposal.StorageCollateral) { - if err := p.addMarketFunds(ctx, waddr, deal); err != nil { - return nil, err - } + // TODO: check StorageCollateral (may be too large (or too small)) + if err := p.full.MarketEnsureAvailable(ctx, waddr, deal.Proposal.StorageCollateral); err != nil { + return nil, err } log.Info("publishing deal") @@ -173,15 +135,20 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) log.Info("fetching data for a deal") mcid := smsg.Cid() err = p.sendSignedResponse(&Response{ - State: api.DealAccepted, - Message: "", + State: api.DealAccepted, + Proposal: deal.ProposalCid, PublishMessage: &mcid, + StorageDeal: &storageDeal, }) if err != nil { return nil, err } + if err := p.disconnect(deal); err != nil { + log.Warnf("closing client connection: %+v", err) + } + return func(deal *MinerDeal) { deal.DealID = resp.DealIDs[0] }, merkledag.FetchGraph(ctx, deal.Ref, p.dag) @@ -190,14 +157,6 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // STAGED func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - err := p.sendSignedResponse(&Response{ - State: api.DealStaged, - Proposal: deal.ProposalCid, - }) - if err != nil { - log.Warnf("Sending deal response failed: %s", err) - } - root, err := p.dag.Get(ctx, deal.Ref) if err != nil { return nil, xerrors.Errorf("failed to get file root for deal: %s", err) @@ -221,21 +180,16 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) if err != nil { return nil, xerrors.Errorf("getting unixfs file size: %w", err) } - if uint64(size) != deal.Proposal.PieceSize { - return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match unixfs file size") + if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize { + return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size") } - pcid, err := cid.Cast(deal.Proposal.PieceRef) - if err != nil { - return nil, err - } - - sectorID, err := p.secst.AddUnixfsPiece(pcid, uf, deal.DealID) + sectorID, err := p.secb.AddUnixfsPiece(ctx, deal.Ref, uf, deal.DealID) if err != nil { return nil, xerrors.Errorf("AddPiece failed: %s", err) } - log.Warnf("New Sector: %d", sectorID) + return func(deal *MinerDeal) { deal.SectorID = sectorID }, nil @@ -243,67 +197,14 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // SEALING -func (p *Provider) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilder.SectorSealingStatus, error) { - status, err := p.secst.WaitSeal(ctx, deal.SectorID) - if err != nil { - return sectorbuilder.SectorSealingStatus{}, err - } - - switch status.State { - case sealing_state.Sealed: - case sealing_state.Failed: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sealing sector %d for deal %s (ref=%s) failed: %s", deal.SectorID, deal.ProposalCid, deal.Ref, status.SealErrorMsg) - case sealing_state.Pending: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'pending' after call to WaitSeal (for sector %d)", deal.SectorID) - case sealing_state.Sealing: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'wait' after call to WaitSeal (for sector %d)", deal.SectorID) - default: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("unknown SealStatusCode: %d", status.SectorID) - } - - return status, nil -} - func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - err := p.sendSignedResponse(&Response{ - State: api.DealSealing, - Proposal: deal.ProposalCid, - }) - if err != nil { - log.Warnf("Sending deal response failed: %s", err) - } - - if err := p.secst.SealSector(ctx, deal.SectorID); err != nil { - return nil, xerrors.Errorf("sealing sector failed: %w", err) - } - - _, err = p.waitSealed(ctx, deal) - if err != nil { - return nil, err - } - // TODO: Spec doesn't say anything about inclusion proofs anywhere - // Not sure what mechanisms prevents miner from storing data that isn't - // clients' data + // TODO: consider waiting for seal to happen return nil, nil } func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - // TODO: Add dealID to commtracker (probably before sealing) - mcid, err := p.commt.WaitCommit(ctx, deal.Proposal.Provider, deal.SectorID) - if err != nil { - log.Warnf("Waiting for sector commitment message: %s", err) - } - - err = p.sendSignedResponse(&Response{ - State: api.DealComplete, - Proposal: deal.ProposalCid, - - CommitMessage: &mcid, - }) - if err != nil { - log.Warnf("Sending deal response failed: %s", err) - } + // TODO: observe sector lifecycle, status, expiration.. return nil, nil } diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index 5931cefc9..e14502287 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -5,11 +5,10 @@ import ( "runtime" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/ipfs/go-cid" inet "github.com/libp2p/go-libp2p-core/network" @@ -26,7 +25,7 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) { cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l) } - log.Errorf("deal %s failed: %s", id, cerr) + log.Warnf("deal %s failed: %s", id, cerr) err := p.sendSignedResponse(&Response{ State: api.DealFailed, @@ -45,21 +44,18 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) { } } -func (p *Provider) readProposal(s inet.Stream) (proposal actors.StorageDealProposal, err error) { - if err := cborrpc.ReadCborRPC(s, &proposal); err != nil { +func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) { + if err := cborutil.ReadCborRPC(s, &proposal); err != nil { log.Errorw("failed to read proposal message", "error", err) return proposal, err } - if err := proposal.Verify(); err != nil { + if err := proposal.DealProposal.Verify(); err != nil { return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) } - // TODO: Validate proposal maybe - // (and signature, obviously) - - if proposal.Provider != p.actor { - log.Errorf("proposal with wrong ProviderAddress: %s", proposal.Provider) + if proposal.DealProposal.Provider != p.actor { + log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider) return proposal, err } @@ -72,7 +68,7 @@ func (p *Provider) sendSignedResponse(resp *Response) error { return xerrors.New("couldn't send response: not connected") } - msg, err := cborrpc.Dump(resp) + msg, err := cborutil.Dump(resp) if err != nil { return xerrors.Errorf("serializing response: %w", err) } @@ -92,7 +88,7 @@ func (p *Provider) sendSignedResponse(resp *Response) error { Signature: sig, } - err = cborrpc.WriteCborRPC(s, signedResponse) + err = cborutil.WriteCborRPC(s, signedResponse) if err != nil { // Assume client disconnected s.Close() @@ -101,6 +97,17 @@ func (p *Provider) sendSignedResponse(resp *Response) error { return err } +func (p *Provider) disconnect(deal MinerDeal) error { + s, ok := p.conns[deal.ProposalCid] + if !ok { + return nil + } + + err := s.Close() + delete(p.conns, deal.ProposalCid) + return err +} + func (p *Provider) getWorker(miner address.Address) (address.Address, error) { getworker := &types.Message{ To: miner, diff --git a/chain/deals/state_store.go b/chain/deals/state_store.go deleted file mode 100644 index ae959146f..000000000 --- a/chain/deals/state_store.go +++ /dev/null @@ -1,142 +0,0 @@ -package deals - -import ( - "bytes" - "github.com/filecoin-project/lotus/lib/cborrpc" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" - "golang.org/x/xerrors" -) - -type StateStore struct { - ds datastore.Datastore -} - -func (st *StateStore) Begin(i cid.Cid, state interface{}) error { - k := datastore.NewKey(i.String()) - has, err := st.ds.Has(k) - if err != nil { - return err - } - if has { - return xerrors.Errorf("Already tracking state for %s", i) - } - - b, err := cborrpc.Dump(state) - if err != nil { - return err - } - - return st.ds.Put(k, b) -} - -func (st *StateStore) End(i cid.Cid) error { - k := datastore.NewKey(i.String()) - has, err := st.ds.Has(k) - if err != nil { - return err - } - if !has { - return xerrors.Errorf("No state for %s", i) - } - return st.ds.Delete(k) -} - -func (st *StateStore) mutate(i cid.Cid, mutator func([]byte) ([]byte, error)) error { - k := datastore.NewKey(i.String()) - has, err := st.ds.Has(k) - if err != nil { - return err - } - if !has { - return xerrors.Errorf("No state for %s", i) - } - - cur, err := st.ds.Get(k) - if err != nil { - return err - } - - mutated, err := mutator(cur) - if err != nil { - return err - } - - return st.ds.Put(k, mutated) -} - -type MinerStateStore struct { - StateStore -} - -func (st *MinerStateStore) MutateMiner(i cid.Cid, mutator func(*MinerDeal) error) error { - return st.mutate(i, minerMutator(mutator)) -} - -func minerMutator(m func(*MinerDeal) error) func([]byte) ([]byte, error) { - return func(in []byte) ([]byte, error) { - deal := new(MinerDeal) - err := cborrpc.ReadCborRPC(bytes.NewReader(in), deal) - if err != nil { - return nil, err - } - - if err := m(deal); err != nil { - return nil, err - } - - return cborrpc.Dump(deal) - } -} - -type ClientStateStore struct { - StateStore -} - -func (st *ClientStateStore) MutateClient(i cid.Cid, mutator func(*ClientDeal) error) error { - return st.mutate(i, clientMutator(mutator)) -} - -func clientMutator(m func(*ClientDeal) error) func([]byte) ([]byte, error) { - return func(in []byte) ([]byte, error) { - deal := new(ClientDeal) - err := cborrpc.ReadCborRPC(bytes.NewReader(in), deal) - if err != nil { - return nil, err - } - - if err := m(deal); err != nil { - return nil, err - } - - return cborrpc.Dump(deal) - } -} - -func (st *ClientStateStore) ListClient() ([]ClientDeal, error) { - var out []ClientDeal - - res, err := st.ds.Query(query.Query{}) - if err != nil { - return nil, err - } - defer res.Close() - - for { - res, ok := res.NextSync() - if !ok { - break - } - - var deal ClientDeal - err := cborrpc.ReadCborRPC(bytes.NewReader(res.Value), &deal) - if err != nil { - return nil, err - } - - out = append(out, deal) - } - - return out, nil -} diff --git a/chain/deals/types.go b/chain/deals/types.go index 27bf31a8b..63e2e45ec 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -5,14 +5,17 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/ipfs/go-cid" ) -const DealProtocolID = "/fil/storage/mk/1.0.0" -const AskProtocolID = "/fil/storage/ask/1.0.0" +const DealProtocolID = "/fil/storage/mk/1.0.1" +const AskProtocolID = "/fil/storage/ask/1.0.1" type Proposal struct { - DealProposal actors.StorageDealProposal + DealProposal *actors.StorageDealProposal + + Piece cid.Cid // Used for retrieving from the client } type Response struct { @@ -25,9 +28,6 @@ type Response struct { // DealAccepted StorageDeal *actors.StorageDeal PublishMessage *cid.Cid - - // DealComplete - CommitMessage *cid.Cid } // TODO: Do we actually need this to be signed? @@ -37,6 +37,15 @@ type SignedResponse struct { Signature *types.Signature } +func (r *SignedResponse) Verify(addr address.Address) error { + b, err := cborutil.Dump(&r.Response) + if err != nil { + return err + } + + return r.Signature.Verify(addr, b) +} + type AskRequest struct { Miner address.Address } diff --git a/chain/events/events.go b/chain/events/events.go index b9decb22f..18287dafe 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -18,8 +18,8 @@ import ( var log = logging.Logger("events") // `curH`-`ts.Height` = `confidence` -type HeightHandler func(ts *types.TipSet, curH uint64) error -type RevertHandler func(ts *types.TipSet) error +type HeightHandler func(ctx context.Context, ts *types.TipSet, curH uint64) error +type RevertHandler func(ctx context.Context, ts *types.TipSet) error type heightHandler struct { confidence int @@ -59,6 +59,7 @@ func NewEvents(ctx context.Context, api eventApi) *Events { heightEvents: heightEvents{ tsc: tsc, + ctx: ctx, gcConfidence: uint64(gcConfidence), heightTriggers: map[uint64]*heightHandler{}, @@ -69,6 +70,7 @@ func NewEvents(ctx context.Context, api eventApi) *Events { calledEvents: calledEvents{ cs: api, tsc: tsc, + ctx: ctx, gcConfidence: uint64(gcConfidence), confQueue: map[triggerH]map[msgH][]*queuedEvent{}, diff --git a/chain/events/events_called.go b/chain/events/events_called.go index 98ea04ec8..94368214e 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -24,7 +24,7 @@ type triggerH = uint64 // `ts` is the tipset, in which the `msg` is included. // `curH`-`ts.Height` = `confidence` -type CalledHandler func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) +type CalledHandler func(msg *types.Message, ts *types.TipSet, curH uint64) (more bool, err error) // CheckFunc is used for atomicity guarantees. If the condition the callbacks // wait for has already happened in tipset `ts` @@ -56,6 +56,7 @@ type queuedEvent struct { type calledEvents struct { cs eventApi tsc *tipSetCache + ctx context.Context gcConfidence uint64 lk sync.Mutex @@ -114,7 +115,7 @@ func (e *calledEvents) handleReverts(ts *types.TipSet) { trigger := e.triggers[event.trigger] - if err := trigger.revert(ts); err != nil { + if err := trigger.revert(e.ctx, ts); err != nil { log.Errorf("reverting chain trigger (call %s.%d() @H %d, called @ %d) failed: %s", event.msg.To, event.msg.Method, ts.Height(), triggerH, err) } } @@ -146,7 +147,11 @@ func (e *calledEvents) checkNewCalls(ts *types.TipSet) { func (e *calledEvents) queueForConfidence(triggerId uint64, msg *types.Message, ts *types.TipSet) { trigger := e.triggers[triggerId] - triggerH := ts.Height() + uint64(trigger.confidence) + + // messages are not applied in the tipset they are included in + appliedH := ts.Height() + 1 + + triggerH := appliedH + uint64(trigger.confidence) byOrigH, ok := e.confQueue[triggerH] if !ok { @@ -154,13 +159,13 @@ func (e *calledEvents) queueForConfidence(triggerId uint64, msg *types.Message, e.confQueue[triggerH] = byOrigH } - byOrigH[ts.Height()] = append(byOrigH[ts.Height()], &queuedEvent{ + byOrigH[appliedH] = append(byOrigH[appliedH], &queuedEvent{ trigger: triggerId, - h: ts.Height(), + h: appliedH, msg: msg, }) - e.revertQueue[ts.Height()] = append(e.revertQueue[ts.Height()], triggerH) // todo: dedupe? + e.revertQueue[appliedH] = append(e.revertQueue[appliedH], triggerH) } func (e *calledEvents) applyWithConfidence(ts *types.TipSet) { diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 2805f38e2..524efd0d5 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -1,8 +1,11 @@ package events import ( + "context" "sync" + "go.opencensus.io/trace" + "github.com/filecoin-project/lotus/chain/types" ) @@ -17,16 +20,29 @@ type heightEvents struct { htTriggerHeights map[triggerH][]triggerId htHeights map[msgH][]triggerId + + ctx context.Context } func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { + ctx, span := trace.StartSpan(e.ctx, "events.HeightHeadChange") + defer span.End() + span.AddAttributes(trace.Int64Attribute("endHeight", int64(app[0].Height()))) + span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev)))) + span.AddAttributes(trace.Int64Attribute("applies", int64(len(app)))) + for _, ts := range rev { // TODO: log error if h below gcconfidence // revert height-based triggers revert := func(h uint64, ts *types.TipSet) { for _, tid := range e.htHeights[h] { - err := e.heightTriggers[tid].revert(ts) + ctx, span := trace.StartSpan(ctx, "events.HeightRevert") + + err := e.heightTriggers[tid].revert(ctx, ts) + + span.End() + if err != nil { log.Errorf("reverting chain trigger (@H %d): %s", h, err) } @@ -74,7 +90,13 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { return err } - if err := hnd.handle(incTs, h); err != nil { + ctx, span := trace.StartSpan(ctx, "events.HeightApply") + span.AddAttributes(trace.BoolAttribute("immediate", false)) + + err = hnd.handle(ctx, incTs, h) + span.End() + + if err != nil { log.Errorf("chain trigger (@H %d, called @ %d) failed: %s", triggerH, ts.Height(), err) } } @@ -125,9 +147,16 @@ func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence } e.lk.Unlock() - if err := hnd(ts, bestH); err != nil { + ctx, span := trace.StartSpan(e.ctx, "events.HeightApply") + span.AddAttributes(trace.BoolAttribute("immediate", true)) + + err = hnd(ctx, ts, bestH) + span.End() + + if err != nil { return err } + e.lk.Lock() bestH = e.tsc.best().Height() } diff --git a/chain/events/events_test.go b/chain/events/events_test.go index f672bc5f6..4c5177e1e 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -46,11 +46,24 @@ func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet func makeTs(t *testing.T, h uint64, msgcid cid.Cid) *types.TipSet { a, _ := address.NewFromString("t00") + b, _ := address.NewFromString("t02") ts, err := types.NewTipSet([]*types.BlockHeader{ { Height: h, Miner: a, + Tickets: []*types.Ticket{{[]byte{byte(h % 2)}}}, + + ParentStateRoot: dummyCid, + Messages: msgcid, + ParentMessageReceipts: dummyCid, + }, + { + Height: h, + Miner: b, + + Tickets: []*types.Ticket{{[]byte{byte((h + 1) % 2)}}}, + ParentStateRoot: dummyCid, Messages: msgcid, ParentMessageReceipts: dummyCid, @@ -183,12 +196,12 @@ func TestAt(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -248,12 +261,12 @@ func TestAtNullTrigger(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, uint64(6), ts.Height()) require.Equal(t, 8, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -282,12 +295,12 @@ func TestAtNullConf(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -323,12 +336,12 @@ func TestAtStart(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -357,12 +370,12 @@ func TestAtStartConfidence(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 11, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -385,16 +398,16 @@ func TestAtChained(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { - return events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { + return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 10, int(ts.Height())) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 10) - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -421,16 +434,16 @@ func TestAtChainedConfidence(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { - return events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { + return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 10, int(ts.Height())) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 10) - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -455,11 +468,11 @@ func TestAtChainedConfidenceNull(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { applied = true require.Equal(t, 6, int(ts.Height())) return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -494,12 +507,13 @@ func TestCalled(t *testing.T) { err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) { + require.Equal(t, false, applied) applied = true appliedMsg = msg appliedTs = ts appliedH = curH return more, nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 20, t0123, 5) @@ -526,28 +540,28 @@ func TestCalled(t *testing.T) { // create additional block so we are above confidence threshold - fcs.advance(0, 1, nil) // H=9 (confidence=3, apply) + fcs.advance(0, 2, nil) // H=10 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false - require.Equal(t, uint64(6), appliedTs.Height()) - require.Equal(t, "bafkqaajq", appliedTs.Blocks()[0].Messages.String()) - require.Equal(t, uint64(9), appliedH) + require.Equal(t, uint64(7), appliedTs.Height()) + require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String()) + require.Equal(t, uint64(10), appliedH) require.Equal(t, t0123, appliedMsg.To) require.Equal(t, uint64(1), appliedMsg.Nonce) require.Equal(t, uint64(5), appliedMsg.Method) // revert some blocks, keep the message - fcs.advance(3, 1, nil) // H=7 (confidence=1) + fcs.advance(3, 1, nil) // H=8 (confidence=1) require.Equal(t, false, applied) require.Equal(t, false, reverted) // revert the message - fcs.advance(2, 1, nil) // H=6, we reverted ts with the msg + fcs.advance(2, 1, nil) // H=7, we reverted ts with the msg require.Equal(t, false, applied) require.Equal(t, true, reverted) @@ -561,7 +575,7 @@ func TestCalled(t *testing.T) { }, }) - fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=7; H=10 (confidence=3) + fcs.advance(0, 5, map[int]cid.Cid{ // (confidence=3) 0: n2msg, }) @@ -569,16 +583,16 @@ func TestCalled(t *testing.T) { require.Equal(t, false, reverted) applied = false - require.Equal(t, uint64(7), appliedTs.Height()) - require.Equal(t, "bafkqaajr", appliedTs.Blocks()[0].Messages.String()) - require.Equal(t, uint64(10), appliedH) + require.Equal(t, uint64(9), appliedTs.Height()) + require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String()) + require.Equal(t, uint64(12), appliedH) require.Equal(t, t0123, appliedMsg.To) require.Equal(t, uint64(2), appliedMsg.Nonce) require.Equal(t, uint64(5), appliedMsg.Method) // revert and apply at different height - fcs.advance(4, 5, map[int]cid.Cid{ // msg at H=8; H=11 (confidence=3) + fcs.advance(4, 6, map[int]cid.Cid{ // (confidence=3) 1: n2msg, }) @@ -590,16 +604,16 @@ func TestCalled(t *testing.T) { reverted = false applied = false - require.Equal(t, uint64(8), appliedTs.Height()) - require.Equal(t, "bafkqaajr", appliedTs.Blocks()[0].Messages.String()) - require.Equal(t, uint64(11), appliedH) + require.Equal(t, uint64(11), appliedTs.Height()) + require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String()) + require.Equal(t, uint64(14), appliedH) require.Equal(t, t0123, appliedMsg.To) require.Equal(t, uint64(2), appliedMsg.Nonce) require.Equal(t, uint64(5), appliedMsg.Method) // call method again - fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=12; H=15 + fcs.advance(0, 5, map[int]cid.Cid{ 0: n2msg, }) @@ -608,7 +622,7 @@ func TestCalled(t *testing.T) { applied = false // send and revert below confidence, then cross confidence - fcs.advance(0, 1, map[int]cid.Cid{ // msg at H=16; H=16 + fcs.advance(0, 2, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 3}, @@ -623,7 +637,7 @@ func TestCalled(t *testing.T) { // test timeout (it's set to 20 in the call to `events.Called` above) - fcs.advance(0, 6, nil) // H=25 + fcs.advance(0, 6, nil) require.Equal(t, false, applied) // not calling timeout as we received messages require.Equal(t, false, reverted) @@ -631,7 +645,7 @@ func TestCalled(t *testing.T) { // test unregistering with more more = false - fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=26; H=29 + fcs.advance(0, 5, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 4}, // this signals we don't want more @@ -643,7 +657,7 @@ func TestCalled(t *testing.T) { require.Equal(t, false, reverted) applied = false - fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=26; H=29 + fcs.advance(0, 5, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 5}, @@ -693,7 +707,7 @@ func TestCalledTimeout(t *testing.T) { require.Equal(t, uint64(20), ts.Height()) require.Equal(t, uint64(23), curH) return false, nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { t.Fatal("revert on timeout") return nil }, 3, 20, t0123, 5) @@ -728,7 +742,7 @@ func TestCalledTimeout(t *testing.T) { require.Equal(t, uint64(20), ts.Height()) require.Equal(t, uint64(23), curH) return false, nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { t.Fatal("revert on timeout") return nil }, 3, 20, t0123, 5) @@ -765,21 +779,21 @@ func TestCalledOrder(t *testing.T) { switch at { case 0: require.Equal(t, uint64(1), msg.Nonce) - require.Equal(t, uint64(3), ts.Height()) + require.Equal(t, uint64(4), ts.Height()) case 1: require.Equal(t, uint64(2), msg.Nonce) - require.Equal(t, uint64(4), ts.Height()) + require.Equal(t, uint64(5), ts.Height()) default: t.Fatal("apply should only get called twice, at: ", at) } at++ return true, nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { switch at { case 2: - require.Equal(t, uint64(4), ts.Height()) + require.Equal(t, uint64(5), ts.Height()) case 3: - require.Equal(t, uint64(3), ts.Height()) + require.Equal(t, uint64(4), ts.Height()) default: t.Fatal("revert should only get called twice, at: ", at) } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index e1452bb91..0d951f1fe 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -279,12 +279,6 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad } fts := store.NewFullTipSet(blks) - fmt.Println("Made a block: ", fts.TipSet().Cids()) - if len(fts.TipSet().Cids()) > 1 { - for _, b := range blks { - fmt.Printf("block %s: %#v\n", b.Cid(), b.Header) - } - } return &MinedTipSet{ TipSet: fts, diff --git a/chain/market/fundmgr.go b/chain/market/fundmgr.go new file mode 100644 index 000000000..9f8068ae2 --- /dev/null +++ b/chain/market/fundmgr.go @@ -0,0 +1,79 @@ +package market + +import ( + "context" + "sync" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/impl/full" +) + +type FundMgr struct { + sm *stmgr.StateManager + mpool full.MpoolAPI + + lk sync.Mutex + available map[address.Address]types.BigInt +} + +func NewFundMgr(sm *stmgr.StateManager, mpool full.MpoolAPI) *FundMgr { + return &FundMgr{ + sm: sm, + mpool: mpool, + + available: map[address.Address]types.BigInt{}, + } +} + +func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error { + fm.lk.Lock() + avail, ok := fm.available[addr] + if !ok { + bal, err := fm.sm.MarketBalance(ctx, addr, nil) + if err != nil { + fm.lk.Unlock() + return err + } + + avail = bal.Available + } + + toAdd := types.NewInt(0) + avail = types.BigSub(avail, amt) + if avail.LessThan(types.NewInt(0)) { + // TODO: some rules around adding more to avoid doing stuff on-chain + // all the time + toAdd = types.BigSub(toAdd, avail) + avail = types.NewInt(0) + } + fm.available[addr] = avail + + fm.lk.Unlock() + + smsg, err := fm.mpool.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: addr, + Value: toAdd, + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.AddBalance, + }) + if err != nil { + return err + } + + _, r, err := fm.sm.WaitForMessage(ctx, smsg.Cid()) + if err != nil { + return err + } + + if r.ExitCode != 0 { + return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.ExitCode) + } + return nil +} diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 1dff70bbd..7a89ab8d5 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -157,7 +157,7 @@ func GetMinerProvingPeriodEnd(ctx context.Context, sm *StateManager, ts *types.T return mas.ProvingPeriodEnd, nil } -func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.SectorInfo, error) { +func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.ChainSectorInfo, error) { var mas actors.StorageMinerActorState _, err := sm.LoadActorState(ctx, maddr, &mas, ts) if err != nil { @@ -167,7 +167,7 @@ func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, return LoadSectorsFromSet(ctx, sm.ChainStore().Blockstore(), mas.ProvingSet) } -func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.SectorInfo, error) { +func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.ChainSectorInfo, error) { var mas actors.StorageMinerActorState _, err := sm.LoadActorState(ctx, maddr, &mas, ts) if err != nil { @@ -193,20 +193,40 @@ func GetMinerSectorSize(ctx context.Context, sm *StateManager, ts *types.TipSet, return minfo.SectorSize, nil } -func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid) ([]*api.SectorInfo, error) { +func GetStorageDeal(ctx context.Context, sm *StateManager, dealId uint64, ts *types.TipSet) (*actors.OnChainDeal, error) { + var state actors.StorageMarketState + if _, err := sm.LoadActorState(ctx, actors.StorageMarketAddress, &state, ts); err != nil { + return nil, err + } + + blks := amt.WrapBlockstore(sm.ChainStore().Blockstore()) + da, err := amt.LoadAMT(blks, state.Deals) + if err != nil { + return nil, err + } + + var ocd actors.OnChainDeal + if err := da.Get(dealId, &ocd); err != nil { + return nil, err + } + + return &ocd, nil +} + +func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid) ([]*api.ChainSectorInfo, error) { blks := amt.WrapBlockstore(bs) a, err := amt.LoadAMT(blks, ssc) if err != nil { return nil, err } - var sset []*api.SectorInfo + var sset []*api.ChainSectorInfo if err := a.ForEach(func(i uint64, v *cbg.Deferred) error { var comms [][]byte if err := cbor.DecodeInto(v.Raw, &comms); err != nil { return err } - sset = append(sset, &api.SectorInfo{ + sset = append(sset, &api.ChainSectorInfo{ SectorID: i, CommR: comms[0], CommD: comms[1], diff --git a/chain/store/weight.go b/chain/store/weight.go index 7c1f80ceb..53c202845 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -37,17 +37,17 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn tpow := types.BigFromBytes(ret.Return) if tpow.GreaterThan(zero) { log2P = int64(tpow.BitLen() - 1) + } else { + // Not really expect to be here ... + return types.EmptyInt, xerrors.Errorf("All power in the net is gone. You network might be disconnected, or the net is dead!") } - out.Add(out, big.NewInt(log2P*256)) + out.Add(out, big.NewInt(log2P<<8)) // (wFunction(totalPowerAtTipset(ts)) * len(ts.blocks) * wRatio_num * 2^8) / (e * wRatio_den) - wRatioNum := int64(1) - wRatioDen := 2 - - eWeight := big.NewInt((log2P * int64(len(ts.Blocks())) * wRatioNum) << 8) - eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch*wRatioDen))) + eWeight := big.NewInt((log2P * int64(len(ts.Blocks())) * build.WRatioNum) << 8) + eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch*build.WRatioDen))) out.Add(out, eWeight) return types.BigInt{Int: out}, nil diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index ddca848e9..5e6c31de1 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -44,7 +44,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha return } - log.Infow("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom()) + log.Debugw("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom()) s.InformNewBlock(msg.GetFrom(), &types.FullBlock{ Header: blk.Header, BlsMessages: bmsgs, diff --git a/chain/sync.go b/chain/sync.go index 55ac65719..80df33d27 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -103,7 +103,7 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) { if from == syncer.self { // TODO: this is kindof a hack... - log.Info("got block from ourselves") + log.Debug("got block from ourselves") if err := syncer.Sync(ctx, fts.TipSet()); err != nil { log.Errorf("failed to sync our own block %s: %+v", fts.TipSet().Cids(), err) @@ -471,6 +471,17 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err } if stateroot != h.ParentStateRoot { + msgs, err := syncer.store.MessagesForTipset(baseTs) + if err != nil { + log.Error("failed to load messages for tipset during tipset state mismatch error: ", err) + } else { + log.Warn("Messages for tipset with mismatching state:") + for i, m := range msgs { + mm := m.VMMessage() + log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params) + } + } + return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot) } @@ -916,7 +927,7 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error } syncer.syncState.SetStage(api.StageSyncComplete) - log.Infow("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids())) + log.Debugw("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids())) return nil } diff --git a/chain/sync_test.go b/chain/sync_test.go index 24b8fbd10..d17b6e853 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" @@ -390,7 +391,7 @@ func TestSyncBadTimestamp(t *testing.T) { base := tu.g.CurTipset tu.g.Timestamper = func(pts *types.TipSet, tl int) uint64 { - return pts.MinTimestamp() + 2 + return pts.MinTimestamp() + (build.BlockDelay / 2) } fmt.Println("BASE: ", base.Cids()) diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 1ede605d4..d65a068d8 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -166,10 +166,12 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { /* Need to check that - h(vrfout) / max(h) < e * minerPower / totalPower + (h(vrfout) + 1) / (max(h) + 1) <= e * minerPower / totalPower max(h) == 2^256-1 which in terms of integer math means: - h(vrfout) * totalPower < e * minerPower * (2^256-1) + (h(vrfout) + 1) * totalPower <= e * minerPower * 2^256 + in 2^256 space, it is equivalent to: + h(vrfout) * totalPower < e * minerPower * 2^256 */ h := sha256.Sum256(eproof) @@ -177,12 +179,12 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { lhs := BigFromBytes(h[:]).Int lhs = lhs.Mul(lhs, totpow.Int) - // rhs = minerPower * 2^256 - minerPower - // rhs = minerPower << 256 - minerPower + // rhs = minerPower * 2^256 + // rhs = minerPower << 256 rhs := new(big.Int).Lsh(mpow.Int, 256) rhs = rhs.Mul(rhs, blocksPerEpoch.Int) - rhs = rhs.Sub(rhs, mpow.Int) + // h(vrfout) * totalPower < e * minerPower * 2^256? return lhs.Cmp(rhs) == -1 } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 9cf6ec913..fa3cddaf4 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "math" - "github.com/ipfs/go-cid" + cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -23,12 +23,12 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { return err } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) if err := t.Miner.MarshalCBOR(w); err != nil { return err } - // t.t.Tickets ([]*types.Ticket) + // t.t.Tickets ([]*types.Ticket) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Tickets)))); err != nil { return err } @@ -38,7 +38,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { } } - // t.t.ElectionProof ([]uint8) + // t.t.ElectionProof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.ElectionProof)))); err != nil { return err } @@ -46,7 +46,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { return err } - // t.t.Parents ([]cid.Cid) + // t.t.Parents ([]cid.Cid) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Parents)))); err != nil { return err } @@ -56,45 +56,45 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { } } - // t.t.ParentWeight (types.BigInt) + // t.t.ParentWeight (types.BigInt) (struct) if err := t.ParentWeight.MarshalCBOR(w); err != nil { return err } - // t.t.Height (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Height)); err != nil { + // t.t.Height (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Height))); err != nil { return err } - // t.t.ParentStateRoot (cid.Cid) + // t.t.ParentStateRoot (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ParentStateRoot); err != nil { return xerrors.Errorf("failed to write cid field t.ParentStateRoot: %w", err) } - // t.t.ParentMessageReceipts (cid.Cid) + // t.t.ParentMessageReceipts (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ParentMessageReceipts); err != nil { return xerrors.Errorf("failed to write cid field t.ParentMessageReceipts: %w", err) } - // t.t.Messages (cid.Cid) + // t.t.Messages (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Messages); err != nil { return xerrors.Errorf("failed to write cid field t.Messages: %w", err) } - // t.t.BLSAggregate (types.Signature) + // t.t.BLSAggregate (types.Signature) (struct) if err := t.BLSAggregate.MarshalCBOR(w); err != nil { return err } - // t.t.Timestamp (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Timestamp)); err != nil { + // t.t.Timestamp (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil { return err } - // t.t.BlockSig (types.Signature) + // t.t.BlockSig (types.Signature) (struct) if err := t.BlockSig.MarshalCBOR(w); err != nil { return err } @@ -116,7 +116,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) { @@ -125,7 +125,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Tickets ([]*types.Ticket) + // t.t.Tickets ([]*types.Ticket) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -151,7 +151,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.Tickets[i] = &v } - // t.t.ElectionProof ([]uint8) + // t.t.ElectionProof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -168,7 +168,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.ElectionProof); err != nil { return err } - // t.t.Parents ([]cid.Cid) + // t.t.Parents ([]cid.Cid) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -193,7 +193,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.Parents[i] = c } - // t.t.ParentWeight (types.BigInt) + // t.t.ParentWeight (types.BigInt) (struct) { @@ -202,7 +202,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Height (uint64) + // t.t.Height (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -211,8 +211,8 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Height = extra - // t.t.ParentStateRoot (cid.Cid) + t.Height = uint64(extra) + // t.t.ParentStateRoot (cid.Cid) (struct) { @@ -224,7 +224,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.ParentStateRoot = c } - // t.t.ParentMessageReceipts (cid.Cid) + // t.t.ParentMessageReceipts (cid.Cid) (struct) { @@ -236,7 +236,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.ParentMessageReceipts = c } - // t.t.Messages (cid.Cid) + // t.t.Messages (cid.Cid) (struct) { @@ -248,7 +248,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.Messages = c } - // t.t.BLSAggregate (types.Signature) + // t.t.BLSAggregate (types.Signature) (struct) { @@ -257,7 +257,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Timestamp (uint64) + // t.t.Timestamp (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -266,8 +266,8 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Timestamp = extra - // t.t.BlockSig (types.Signature) + t.Timestamp = uint64(extra) + // t.t.BlockSig (types.Signature) (struct) { @@ -288,7 +288,7 @@ func (t *Ticket) MarshalCBOR(w io.Writer) error { return err } - // t.t.VRFProof ([]uint8) + // t.t.VRFProof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.VRFProof)))); err != nil { return err } @@ -313,7 +313,7 @@ func (t *Ticket) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.VRFProof ([]uint8) + // t.t.VRFProof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -342,42 +342,42 @@ func (t *Message) MarshalCBOR(w io.Writer) error { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) if err := t.From.MarshalCBOR(w); err != nil { return err } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) if err := t.Value.MarshalCBOR(w); err != nil { return err } - // t.t.GasPrice (types.BigInt) + // t.t.GasPrice (types.BigInt) (struct) if err := t.GasPrice.MarshalCBOR(w); err != nil { return err } - // t.t.GasLimit (types.BigInt) + // t.t.GasLimit (types.BigInt) (struct) if err := t.GasLimit.MarshalCBOR(w); err != nil { return err } - // t.t.Method (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Method)); err != nil { + // t.t.Method (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { return err } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Params)))); err != nil { return err } @@ -402,7 +402,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -411,7 +411,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) { @@ -420,7 +420,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Nonce (uint64) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -429,8 +429,8 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra - // t.t.Value (types.BigInt) + t.Nonce = uint64(extra) + // t.t.Value (types.BigInt) (struct) { @@ -439,7 +439,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.GasPrice (types.BigInt) + // t.t.GasPrice (types.BigInt) (struct) { @@ -448,7 +448,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.GasLimit (types.BigInt) + // t.t.GasLimit (types.BigInt) (struct) { @@ -457,7 +457,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Method (uint64) + // t.t.Method (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -466,8 +466,8 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Method = extra - // t.t.Params ([]uint8) + t.Method = uint64(extra) + // t.t.Params ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -496,12 +496,12 @@ func (t *SignedMessage) MarshalCBOR(w io.Writer) error { return err } - // t.t.Message (types.Message) + // t.t.Message (types.Message) (struct) if err := t.Message.MarshalCBOR(w); err != nil { return err } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) if err := t.Signature.MarshalCBOR(w); err != nil { return err } @@ -523,7 +523,7 @@ func (t *SignedMessage) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Message (types.Message) + // t.t.Message (types.Message) (struct) { @@ -532,7 +532,7 @@ func (t *SignedMessage) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) { @@ -553,13 +553,13 @@ func (t *MsgMeta) MarshalCBOR(w io.Writer) error { return err } - // t.t.BlsMessages (cid.Cid) + // t.t.BlsMessages (cid.Cid) (struct) if err := cbg.WriteCid(w, t.BlsMessages); err != nil { return xerrors.Errorf("failed to write cid field t.BlsMessages: %w", err) } - // t.t.SecpkMessages (cid.Cid) + // t.t.SecpkMessages (cid.Cid) (struct) if err := cbg.WriteCid(w, t.SecpkMessages); err != nil { return xerrors.Errorf("failed to write cid field t.SecpkMessages: %w", err) @@ -583,7 +583,7 @@ func (t *MsgMeta) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.BlsMessages (cid.Cid) + // t.t.BlsMessages (cid.Cid) (struct) { @@ -595,7 +595,7 @@ func (t *MsgMeta) UnmarshalCBOR(r io.Reader) error { t.BlsMessages = c } - // t.t.SecpkMessages (cid.Cid) + // t.t.SecpkMessages (cid.Cid) (struct) { @@ -619,12 +619,12 @@ func (t *SignedVoucher) MarshalCBOR(w io.Writer) error { return err } - // t.t.TimeLock (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.TimeLock)); err != nil { + // t.t.TimeLock (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TimeLock))); err != nil { return err } - // t.t.SecretPreimage ([]uint8) + // t.t.SecretPreimage ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.SecretPreimage)))); err != nil { return err } @@ -632,32 +632,32 @@ func (t *SignedVoucher) MarshalCBOR(w io.Writer) error { return err } - // t.t.Extra (types.ModVerifyParams) + // t.t.Extra (types.ModVerifyParams) (struct) if err := t.Extra.MarshalCBOR(w); err != nil { return err } - // t.t.Lane (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Lane)); err != nil { + // t.t.Lane (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { return err } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } - // t.t.Amount (types.BigInt) + // t.t.Amount (types.BigInt) (struct) if err := t.Amount.MarshalCBOR(w); err != nil { return err } - // t.t.MinCloseHeight (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.MinCloseHeight)); err != nil { + // t.t.MinCloseHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinCloseHeight))); err != nil { return err } - // t.t.Merges ([]types.Merge) + // t.t.Merges ([]types.Merge) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Merges)))); err != nil { return err } @@ -667,7 +667,7 @@ func (t *SignedVoucher) MarshalCBOR(w io.Writer) error { } } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) if err := t.Signature.MarshalCBOR(w); err != nil { return err } @@ -689,7 +689,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.TimeLock (uint64) + // t.t.TimeLock (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -698,8 +698,8 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.TimeLock = extra - // t.t.SecretPreimage ([]uint8) + t.TimeLock = uint64(extra) + // t.t.SecretPreimage ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -716,7 +716,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.SecretPreimage); err != nil { return err } - // t.t.Extra (types.ModVerifyParams) + // t.t.Extra (types.ModVerifyParams) (struct) { @@ -737,7 +737,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Lane (uint64) + // t.t.Lane (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -746,8 +746,8 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Lane = extra - // t.t.Nonce (uint64) + t.Lane = uint64(extra) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -756,8 +756,8 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra - // t.t.Amount (types.BigInt) + t.Nonce = uint64(extra) + // t.t.Amount (types.BigInt) (struct) { @@ -766,7 +766,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { } } - // t.t.MinCloseHeight (uint64) + // t.t.MinCloseHeight (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -775,8 +775,8 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.MinCloseHeight = extra - // t.t.Merges ([]types.Merge) + t.MinCloseHeight = uint64(extra) + // t.t.Merges ([]types.Merge) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -802,7 +802,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { t.Merges[i] = v } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) { @@ -835,17 +835,17 @@ func (t *ModVerifyParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Actor (address.Address) + // t.t.Actor (address.Address) (struct) if err := t.Actor.MarshalCBOR(w); err != nil { return err } - // t.t.Method (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Method)); err != nil { + // t.t.Method (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { return err } - // t.t.Data ([]uint8) + // t.t.Data ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Data)))); err != nil { return err } @@ -870,7 +870,7 @@ func (t *ModVerifyParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Actor (address.Address) + // t.t.Actor (address.Address) (struct) { @@ -879,7 +879,7 @@ func (t *ModVerifyParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Method (uint64) + // t.t.Method (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -888,8 +888,8 @@ func (t *ModVerifyParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Method = extra - // t.t.Data ([]uint8) + t.Method = uint64(extra) + // t.t.Data ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -918,13 +918,13 @@ func (t *Merge) MarshalCBOR(w io.Writer) error { return err } - // t.t.Lane (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Lane)); err != nil { + // t.t.Lane (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { return err } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } return nil @@ -945,7 +945,7 @@ func (t *Merge) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Lane (uint64) + // t.t.Lane (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -954,8 +954,8 @@ func (t *Merge) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Lane = extra - // t.t.Nonce (uint64) + t.Lane = uint64(extra) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -964,7 +964,7 @@ func (t *Merge) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra + t.Nonce = uint64(extra) return nil } @@ -977,24 +977,24 @@ func (t *Actor) MarshalCBOR(w io.Writer) error { return err } - // t.t.Code (cid.Cid) + // t.t.Code (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Code); err != nil { return xerrors.Errorf("failed to write cid field t.Code: %w", err) } - // t.t.Head (cid.Cid) + // t.t.Head (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Head); err != nil { return xerrors.Errorf("failed to write cid field t.Head: %w", err) } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } - // t.t.Balance (types.BigInt) + // t.t.Balance (types.BigInt) (struct) if err := t.Balance.MarshalCBOR(w); err != nil { return err } @@ -1016,7 +1016,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Code (cid.Cid) + // t.t.Code (cid.Cid) (struct) { @@ -1028,7 +1028,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { t.Code = c } - // t.t.Head (cid.Cid) + // t.t.Head (cid.Cid) (struct) { @@ -1040,7 +1040,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { t.Head = c } - // t.t.Nonce (uint64) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1049,8 +1049,8 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra - // t.t.Balance (types.BigInt) + t.Nonce = uint64(extra) + // t.t.Balance (types.BigInt) (struct) { @@ -1071,12 +1071,12 @@ func (t *MessageReceipt) MarshalCBOR(w io.Writer) error { return err } - // t.t.ExitCode (uint8) + // t.t.ExitCode (uint8) (uint8) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ExitCode))); err != nil { return err } - // t.t.Return ([]uint8) + // t.t.Return ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Return)))); err != nil { return err } @@ -1084,7 +1084,7 @@ func (t *MessageReceipt) MarshalCBOR(w io.Writer) error { return err } - // t.t.GasUsed (types.BigInt) + // t.t.GasUsed (types.BigInt) (struct) if err := t.GasUsed.MarshalCBOR(w); err != nil { return err } @@ -1106,7 +1106,7 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.ExitCode (uint8) + // t.t.ExitCode (uint8) (uint8) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1119,7 +1119,7 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("integer in input was too large for uint8 field") } t.ExitCode = uint8(extra) - // t.t.Return ([]uint8) + // t.t.Return ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1136,7 +1136,7 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Return); err != nil { return err } - // t.t.GasUsed (types.BigInt) + // t.t.GasUsed (types.BigInt) (struct) { @@ -1157,12 +1157,12 @@ func (t *BlockMsg) MarshalCBOR(w io.Writer) error { return err } - // t.t.Header (types.BlockHeader) + // t.t.Header (types.BlockHeader) (struct) if err := t.Header.MarshalCBOR(w); err != nil { return err } - // t.t.BlsMessages ([]cid.Cid) + // t.t.BlsMessages ([]cid.Cid) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.BlsMessages)))); err != nil { return err } @@ -1172,7 +1172,7 @@ func (t *BlockMsg) MarshalCBOR(w io.Writer) error { } } - // t.t.SecpkMessages ([]cid.Cid) + // t.t.SecpkMessages ([]cid.Cid) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.SecpkMessages)))); err != nil { return err } @@ -1199,7 +1199,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Header (types.BlockHeader) + // t.t.Header (types.BlockHeader) (struct) { @@ -1220,7 +1220,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { } } - // t.t.BlsMessages ([]cid.Cid) + // t.t.BlsMessages ([]cid.Cid) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1245,7 +1245,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { t.BlsMessages[i] = c } - // t.t.SecpkMessages ([]cid.Cid) + // t.t.SecpkMessages ([]cid.Cid) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1282,12 +1282,12 @@ func (t *SignedStorageAsk) MarshalCBOR(w io.Writer) error { return err } - // t.t.Ask (types.StorageAsk) + // t.t.Ask (types.StorageAsk) (struct) if err := t.Ask.MarshalCBOR(w); err != nil { return err } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) if err := t.Signature.MarshalCBOR(w); err != nil { return err } @@ -1309,7 +1309,7 @@ func (t *SignedStorageAsk) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Ask (types.StorageAsk) + // t.t.Ask (types.StorageAsk) (struct) { @@ -1330,7 +1330,7 @@ func (t *SignedStorageAsk) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) { @@ -1363,33 +1363,33 @@ func (t *StorageAsk) MarshalCBOR(w io.Writer) error { return err } - // t.t.Price (types.BigInt) + // t.t.Price (types.BigInt) (struct) if err := t.Price.MarshalCBOR(w); err != nil { return err } - // t.t.MinPieceSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.MinPieceSize)); err != nil { + // t.t.MinPieceSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinPieceSize))); err != nil { return err } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) if err := t.Miner.MarshalCBOR(w); err != nil { return err } - // t.t.Timestamp (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Timestamp)); err != nil { + // t.t.Timestamp (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil { return err } - // t.t.Expiry (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Expiry)); err != nil { + // t.t.Expiry (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Expiry))); err != nil { return err } - // t.t.SeqNo (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SeqNo)); err != nil { + // t.t.SeqNo (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SeqNo))); err != nil { return err } return nil @@ -1410,7 +1410,7 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Price (types.BigInt) + // t.t.Price (types.BigInt) (struct) { @@ -1419,7 +1419,7 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { } } - // t.t.MinPieceSize (uint64) + // t.t.MinPieceSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1428,8 +1428,8 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.MinPieceSize = extra - // t.t.Miner (address.Address) + t.MinPieceSize = uint64(extra) + // t.t.Miner (address.Address) (struct) { @@ -1438,7 +1438,7 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Timestamp (uint64) + // t.t.Timestamp (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1447,8 +1447,8 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Timestamp = extra - // t.t.Expiry (uint64) + t.Timestamp = uint64(extra) + // t.t.Expiry (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1457,8 +1457,8 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Expiry = extra - // t.t.SeqNo (uint64) + t.Expiry = uint64(extra) + // t.t.SeqNo (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1467,6 +1467,121 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SeqNo = extra + t.SeqNo = uint64(extra) + return nil +} + +func (t *ExpTipSet) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Cids ([]cid.Cid) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Cids)))); err != nil { + return err + } + for _, v := range t.Cids { + if err := cbg.WriteCid(w, v); err != nil { + return xerrors.Errorf("failed writing cid field t.Cids: %w", err) + } + } + + // t.t.Blocks ([]*types.BlockHeader) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Blocks)))); err != nil { + return err + } + for _, v := range t.Blocks { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.t.Height (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Height))); err != nil { + return err + } + return nil +} + +func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Cids ([]cid.Cid) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Cids: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Cids = make([]cid.Cid, extra) + } + for i := 0; i < int(extra); i++ { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("reading cid field t.Cids failed: %w", err) + } + t.Cids[i] = c + } + + // t.t.Blocks ([]*types.BlockHeader) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Blocks: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Blocks = make([]*BlockHeader, extra) + } + for i := 0; i < int(extra); i++ { + + var v BlockHeader + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Blocks[i] = &v + } + + // t.t.Height (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Height = uint64(extra) return nil } diff --git a/chain/types/tipset.go b/chain/types/tipset.go index 40b2b2b0b..733e29685 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -4,10 +4,12 @@ import ( "bytes" "encoding/json" "fmt" + "io" "sort" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + cbg "github.com/whyrusleeping/cbor-gen" ) var log = logging.Logger("types") @@ -20,14 +22,14 @@ type TipSet struct { // why didnt i just export the fields? Because the struct has methods with the // same names already -type expTipSet struct { +type ExpTipSet struct { Cids []cid.Cid Blocks []*BlockHeader Height uint64 } func (ts *TipSet) MarshalJSON() ([]byte, error) { - return json.Marshal(expTipSet{ + return json.Marshal(ExpTipSet{ Cids: ts.cids, Blocks: ts.blks, Height: ts.height, @@ -35,7 +37,7 @@ func (ts *TipSet) MarshalJSON() ([]byte, error) { } func (ts *TipSet) UnmarshalJSON(b []byte) error { - var ets expTipSet + var ets ExpTipSet if err := json.Unmarshal(b, &ets); err != nil { return err } @@ -50,6 +52,34 @@ func (ts *TipSet) UnmarshalJSON(b []byte) error { return nil } +func (ts *TipSet) MarshalCBOR(w io.Writer) error { + if ts == nil { + _, err := w.Write(cbg.CborNull) + return err + } + return (&ExpTipSet{ + Cids: ts.cids, + Blocks: ts.blks, + Height: ts.height, + }).MarshalCBOR(w) +} + +func (ts *TipSet) UnmarshalCBOR(r io.Reader) error { + var ets ExpTipSet + if err := ets.UnmarshalCBOR(r); err != nil { + return err + } + + ots, err := NewTipSet(ets.Blocks) + if err != nil { + return err + } + + *ts = *ots + + return nil +} + func tipsetSortFunc(blks []*BlockHeader) func(i, j int) bool { return func(i, j int) bool { ti := blks[i].LastTicket() diff --git a/chain/types/voucher.go b/chain/types/voucher.go index 7855b21a2..1a510d07e 100644 --- a/chain/types/voucher.go +++ b/chain/types/voucher.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "github.com/filecoin-project/lotus/chain/address" + cborrpc "github.com/filecoin-project/lotus/lib/cborutil" cbor "github.com/ipfs/go-ipld-cbor" ) @@ -46,13 +47,13 @@ func (sv *SignedVoucher) EncodedString() (string, error) { func (sv *SignedVoucher) Equals(other *SignedVoucher) bool { // TODO: make this less bad - selfB, err := cbor.DumpObject(sv) + selfB, err := cborrpc.Dump(sv) if err != nil { log.Errorf("SignedVoucher.Equals: dump self: %s", err) return false } - otherB, err := cbor.DumpObject(other) + otherB, err := cborrpc.Dump(other) if err != nil { log.Errorf("SignedVoucher.Equals: dump other: %s", err) return false diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index decc590b7..3fa02d4c1 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -2,6 +2,7 @@ package vm import ( "bytes" + "encoding/hex" "fmt" "reflect" @@ -43,7 +44,8 @@ func (inv *invoker) Invoke(act *types.Actor, vmctx types.VMContext, method uint6 code, ok := inv.builtInCode[act.Code] if !ok { - return nil, aerrors.Newf(255, "no code for actor %s", act.Code) + log.Errorf("no code for actor %s", act.Code) + return nil, aerrors.Newf(255, "no code for actor %s(%d)(%s)", act.Code, method, hex.EncodeToString(params)) } if method >= uint64(len(code)) || code[method] == nil { return nil, aerrors.Newf(255, "no method %d on actor", method) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 38831043b..d090e5f8d 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -462,7 +462,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, msg *types.Message) (*ApplyRet, return nil, xerrors.Errorf("fatal error: %w", actorErr) } if actorErr != nil { - log.Warnf("[from=%s,n=%d,h=%d] Send actor error: %+v", msg.From, msg.Nonce, vm.blockHeight, actorErr) + log.Warnf("[from=%s,to=%s,n=%d,m=%d,h=%d] Send actor error: %+v", msg.From, msg.To, msg.Nonce, msg.Method, vm.blockHeight, actorErr) } var errcode uint8 diff --git a/cli/client.go b/cli/client.go index f61be20f0..f3602624c 100644 --- a/cli/client.go +++ b/cli/client.go @@ -2,14 +2,17 @@ package cli import ( "fmt" + "os" "path/filepath" "strconv" + "text/tabwriter" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + lapi "github.com/filecoin-project/lotus/api" actors "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" @@ -25,6 +28,7 @@ var clientCmd = &cli.Command{ clientFindCmd, clientRetrieveCmd, clientQueryAskCmd, + clientListDeals, }, } @@ -229,11 +233,12 @@ var clientRetrieveCmd = &cli.Command{ order := offers[0].Order() order.Client = payer - err = api.ClientRetrieve(ctx, order, cctx.Args().Get(1)) - if err == nil { - fmt.Println("Success") + if err := api.ClientRetrieve(ctx, order, cctx.Args().Get(1)); err != nil { + return err } - return err + + fmt.Println("Success") + return nil }, } @@ -325,3 +330,28 @@ var clientQueryAskCmd = &cli.Command{ return nil }, } + +var clientListDeals = &cli.Command{ + Name: "list-deals", + Usage: "List storage market deals", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + deals, err := api.ClientListDeals(ctx) + if err != nil { + return err + } + + w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + fmt.Fprintf(w, "DealCid\tProvider\tState\tPieceRef\tSize\tPrice\tDuration\n") + for _, d := range deals { + fmt.Fprintf(w, "%s\t%s\t%s\t%x\t%d\t%s\t%d\n", d.ProposalCid, d.Provider, lapi.DealStates[d.State], d.PieceRef, d.Size, d.PricePerEpoch, d.Duration) + } + return w.Flush() + }, +} diff --git a/cmd/lotus-storage-miner/commitments.go b/cmd/lotus-storage-miner/commitments.go deleted file mode 100644 index a7825deb1..000000000 --- a/cmd/lotus-storage-miner/commitments.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "fmt" - lcli "github.com/filecoin-project/lotus/cli" - - "gopkg.in/urfave/cli.v2" -) - -var commitmentsCmd = &cli.Command{ - Name: "commitments", - Usage: "interact with commitment tracker", - Subcommands: []*cli.Command{ - commitmentsListCmd, - }, -} - -var commitmentsListCmd = &cli.Command{ - Name: "list", - Usage: "List tracked sector commitments", - Action: func(cctx *cli.Context) error { - api, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := lcli.ReqContext(cctx) - - comms, err := api.CommitmentsList(ctx) - if err != nil { - return err - } - - for _, comm := range comms { - fmt.Printf("%s:%d msg:%s, deals: %v\n", comm.Miner, comm.SectorID, comm.CommitMsg, comm.DealIDs) - } - - return nil - }, -} diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 3ce7c0ff9..074540faf 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -6,7 +6,6 @@ import ( "gopkg.in/urfave/cli.v2" - sectorstate "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" @@ -43,8 +42,7 @@ var infoCmd = &cli.Command{ return err } - size, unit := getSizeAndUnit(sizeByte) - fmt.Printf("Sector Size: %g %s\n", size, unit) + fmt.Printf("Sector Size: %s\n", sizeStr(sizeByte)) pow, err := api.StateMinerPower(ctx, maddr, nil) if err != nil { @@ -54,15 +52,19 @@ var infoCmd = &cli.Command{ percI := types.BigDiv(types.BigMul(pow.MinerPower, types.NewInt(1000)), pow.TotalPower) fmt.Printf("Power: %s / %s (%0.2f%%)\n", pow.MinerPower, pow.TotalPower, float64(percI.Int64())/1000*100) + // TODO: indicate whether the post worker is in use + wstat, err := nodeApi.WorkerStats(ctx) + if err != nil { + return err + } + fmt.Printf("Worker use: %d / %d (+%d)", wstat.Total-wstat.Reserved-wstat.Free, wstat.Total, wstat.Reserved) + sinfo, err := sectorsInfo(ctx, nodeApi) if err != nil { return err } - fmt.Println("Sealed Sectors:\t", sinfo.SealedCount) - fmt.Println("Sealing Sectors:\t", sinfo.SealingCount) - fmt.Println("Pending Sectors:\t", sinfo.PendingCount) - fmt.Println("Failed Sectors:\t", sinfo.FailedCount) + fmt.Println(sinfo) // TODO: grab actr state / info // * Sealed sectors (count / bytes) @@ -71,34 +73,26 @@ var infoCmd = &cli.Command{ }, } -var Units = []string{"B", "KiB", "MiB", "GiB"} +var Units = []string{"B", "KiB", "MiB", "GiB", "TiB"} -func getSizeAndUnit(size uint64) (float64, string) { +func sizeStr(size uint64) string { i := 0 unitSize := float64(size) - for unitSize >= 1024 && i < len(Units) - 1 { + for unitSize >= 1024 && i < len(Units)-1 { unitSize = unitSize / 1024 i++ } - return unitSize, Units[i] + return fmt.Sprintf("%g %s", unitSize, Units[i]) } -type SectorsInfo struct { - TotalCount int - SealingCount int - FailedCount int - SealedCount int - PendingCount int -} - -func sectorsInfo(ctx context.Context, napi api.StorageMiner) (*SectorsInfo, error) { +func sectorsInfo(ctx context.Context, napi api.StorageMiner) (map[string]int, error) { sectors, err := napi.SectorsList(ctx) if err != nil { return nil, err } - out := SectorsInfo{ - TotalCount: len(sectors), + out := map[string]int{ + "Total": len(sectors), } for _, s := range sectors { st, err := napi.SectorsStatus(ctx, s) @@ -106,18 +100,8 @@ func sectorsInfo(ctx context.Context, napi api.StorageMiner) (*SectorsInfo, erro return nil, err } - switch st.State { - case sectorstate.Sealed: - out.SealedCount++ - case sectorstate.Pending: - out.PendingCount++ - case sectorstate.Sealing: - out.SealingCount++ - case sectorstate.Failed: - out.FailedCount++ - case sectorstate.Unknown: - } + out[api.SectorStateStr(st.State)]++ } - return &out, nil + return out, nil } diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index b6aad1d73..f8bf2aac1 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -26,7 +26,6 @@ func main() { infoCmd, storeGarbageCmd, sectorsCmd, - commitmentsCmd, } jaeger := tracing.SetupJaegerTracing("lotus") defer func() { diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index d0806eb16..c8a10baac 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -8,13 +8,12 @@ import ( "os/signal" "syscall" - "github.com/filecoin-project/lotus/build" - "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/auth" "github.com/filecoin-project/lotus/lib/jsonrpc" @@ -32,12 +31,20 @@ var runCmd = &cli.Command{ Name: "api", Value: "2345", }, + &cli.BoolFlag{ + Name: "enable-gpu-proving", + Usage: "Enable use of GPU for mining operations", + }, }, Action: func(cctx *cli.Context) error { if err := build.GetParams(true); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } + if !cctx.Bool("enable-gpu-proving") { + os.Setenv("BELLMAN_NO_GPU", "true") + } + nodeApi, ncloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err @@ -77,7 +84,7 @@ var runCmd = &cli.Command{ } return lr.SetAPIEndpoint(apima) }), - node.Override(new(*sectorbuilder.SectorBuilderConfig), modules.SectorBuilderConfig(storageRepoPath)), + node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(storageRepoPath, 5)), // TODO: grab worker count from config node.Override(new(api.FullNode), nodeApi), ) if err != nil { diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index d49384619..627e11c14 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -6,6 +6,7 @@ import ( "gopkg.in/urfave/cli.v2" + "github.com/filecoin-project/lotus/api" lcli "github.com/filecoin-project/lotus/cli" ) @@ -60,14 +61,15 @@ var sectorsStatusCmd = &cli.Command{ } fmt.Printf("SectorID:\t%d\n", status.SectorID) - fmt.Printf("Status:\t%s\n", status.State.String()) - fmt.Printf("SealErrorMsg:\t%q\n", status.SealErrorMsg) + fmt.Printf("Status:\t%s\n", api.SectorStateStr(status.State)) fmt.Printf("CommD:\t\t%x\n", status.CommD) fmt.Printf("CommR:\t\t%x\n", status.CommR) fmt.Printf("Ticket:\t\t%x\n", status.Ticket.TicketBytes) fmt.Printf("TicketH:\t\t%d\n", status.Ticket.BlockHeight) + fmt.Printf("Seed:\t\t%x\n", status.Seed.TicketBytes) + fmt.Printf("SeedH:\t\t%d\n", status.Seed.BlockHeight) fmt.Printf("Proof:\t\t%x\n", status.Proof) - fmt.Printf("Pieces:\t\t%v\n", status.Pieces) + fmt.Printf("Deals:\t\t%v\n", status.Deals) return nil }, } diff --git a/extern/go-bls-sigs b/extern/go-bls-sigs index 10fef3cbf..98479d3c7 160000 --- a/extern/go-bls-sigs +++ b/extern/go-bls-sigs @@ -1 +1 @@ -Subproject commit 10fef3cbfa8670d9e9cdffb0cd8dfd4f6f6fef07 +Subproject commit 98479d3c79620f18783da0c2c6a15f8b8eb4fa2e diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index 692725ff2..e198d9050 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit 692725ff21919ce9c9df9ea87621b0c1e6a9746c +Subproject commit e198d9050b5dde6c19bef3593506597afe00b6cb diff --git a/gen/main.go b/gen/main.go index 127336a0b..b2b0140e7 100644 --- a/gen/main.go +++ b/gen/main.go @@ -6,9 +6,13 @@ import ( gen "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/paych" + "github.com/filecoin-project/lotus/retrieval" + "github.com/filecoin-project/lotus/storage" ) func main() { @@ -26,6 +30,41 @@ func main() { types.BlockMsg{}, types.SignedStorageAsk{}, types.StorageAsk{}, + types.ExpTipSet{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = gen.WriteTupleEncodersToFile("./paych/cbor_gen.go", "paych", + paych.VoucherInfo{}, + paych.ChannelInfo{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = gen.WriteTupleEncodersToFile("./api/cbor_gen.go", "api", + api.PaymentInfo{}, + api.SealedRef{}, + api.SealedRefs{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = gen.WriteTupleEncodersToFile("./retrieval/cbor_gen.go", "retrieval", + retrieval.RetParams{}, + + retrieval.Query{}, + retrieval.QueryResponse{}, + retrieval.Unixfs0Offer{}, + retrieval.DealProposal{}, + retrieval.DealResponse{}, + retrieval.Block{}, ) if err != nil { fmt.Println(err) @@ -50,7 +89,8 @@ func main() { actors.AccountActorState{}, actors.StorageMinerActorState{}, actors.StorageMinerConstructorParams{}, - actors.OnChainSealVerifyInfo{}, + actors.SectorPreCommitInfo{}, + actors.PreCommittedSector{}, actors.MinerInfo{}, actors.SubmitPoStParams{}, actors.PaymentVerifyParams{}, @@ -87,6 +127,8 @@ func main() { actors.ActivateStorageDealsParams{}, actors.ProcessStorageDealsPaymentParams{}, actors.OnChainDeal{}, + actors.ComputeDataCommitmentParams{}, + actors.SectorProveCommitInfo{}, ) if err != nil { fmt.Println(err) @@ -107,4 +149,15 @@ func main() { fmt.Println(err) os.Exit(1) } + + err = gen.WriteTupleEncodersToFile("./storage/cbor_gen.go", "storage", + storage.SealTicket{}, + storage.SealSeed{}, + storage.Piece{}, + storage.SectorInfo{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } } diff --git a/go.mod b/go.mod index 67f47edc4..8e62267a7 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20191001154818-b4b5288fcb86 + github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.0 @@ -84,7 +84,7 @@ require ( go.uber.org/zap v1.10.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect - golang.org/x/time v0.0.0-20181108054448-85acf8d2951c + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 google.golang.org/api v0.9.0 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 diff --git a/go.sum b/go.sum index fd9e20aa9..7e29dcdf1 100644 --- a/go.sum +++ b/go.sum @@ -13,10 +13,8 @@ github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= -github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= @@ -30,8 +28,6 @@ github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -75,7 +71,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16 h1:NzojcJU1VbS6zdLG13JMYis/cQy/MrN3rxmZRq56jKA= @@ -88,35 +83,16 @@ github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1/go.mod h1:0eHX/BVySxPc6SE2mZRoppGq7qcEagxdmQnA3dzork8= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= -github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= -github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= -github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= -github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= -github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= -github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= -github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= -github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= -github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= -github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -126,29 +102,11 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= -github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= -github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg= -github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= @@ -163,7 +121,6 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -174,7 +131,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -291,8 +247,6 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -300,13 +254,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muMD06dzLiahtGM1eouRNOzVV7tdQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -433,10 +382,8 @@ github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= -github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA= @@ -445,7 +392,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -453,7 +401,6 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -465,17 +412,9 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= -github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= @@ -503,30 +442,22 @@ github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/neelance/parallel v0.0.0-20160708114440-4de9ce63d14c/go.mod h1:eTBvSIlRgLo+CNFFQRQTwUGTZOEdvXIKeZS/xG+D2yU= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc= github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M= github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -541,32 +472,16 @@ github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3/go.mod h1:AIBPxLCkKUFc2ZkjCXzs/Kk9OUhQLw/Zicdd0Rhqz2U= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -576,27 +491,17 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= -github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81/go.mod h1:xIvvI5FiHLxhv8prbzVpaMHaaGPFPFQSuTcxC91ryOo= -github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= -github.com/sourcegraph/go-langserver v2.0.0+incompatible/go.mod h1:bBMjfpzEHd6ijPRoQ7f+knFfw+e8R+W158/MsqAy77c= -github.com/sourcegraph/jsonrpc2 v0.0.0-20190106185902-35a74f039c6a/go.mod h1:eESpbCslcLDs8j2D7IEdGVgul7xuk9odqDTaor30IUU= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -606,19 +511,14 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -628,8 +528,8 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM= github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= -github.com/whyrusleeping/cbor-gen v0.0.0-20191001154818-b4b5288fcb86 h1:cE8mRdI9JQAheSlIAkjJIpdAOPjYOzxSADaro6LNHnY= -github.com/whyrusleeping/cbor-gen v0.0.0-20191001154818-b4b5288fcb86/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679 h1:ct50KYdZHcdOnTAuSgppw5MZKTa3RA63FX28m0l9Aeg= +github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= @@ -676,13 +576,11 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -692,11 +590,9 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -710,10 +606,11 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -723,7 +620,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -736,7 +632,6 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -744,37 +639,30 @@ golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190813142322-97f12d73768f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -795,27 +683,27 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -824,7 +712,3 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= -mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/lib/cborrpc/rpc.go b/lib/cborutil/rpc.go similarity index 84% rename from lib/cborrpc/rpc.go rename to lib/cborutil/rpc.go index 901108457..06af9a156 100644 --- a/lib/cborrpc/rpc.go +++ b/lib/cborutil/rpc.go @@ -1,4 +1,4 @@ -package cborrpc +package cborutil import ( "bytes" @@ -66,3 +66,15 @@ func AsIpld(obj interface{}) (ipld.Node, error) { } return cbor.WrapObject(obj, math.MaxUint64, -1) } + +func Equals(a cbg.CBORMarshaler, b cbg.CBORMarshaler) (bool, error) { + ab, err := Dump(a) + if err != nil { + return false, err + } + bb, err := Dump(b) + if err != nil { + return false, err + } + return bytes.Equal(ab, bb), nil +} diff --git a/lib/jsonrpc/client.go b/lib/jsonrpc/client.go index 81cf4ec0a..45b52c0e0 100644 --- a/lib/jsonrpc/client.go +++ b/lib/jsonrpc/client.go @@ -177,9 +177,9 @@ func (c *client) makeOutChan(ctx context.Context, ftyp reflect.Type, valOut int) front := buf.Front() bufLk.Unlock() - cases := []reflect.SelectCase{ + cases := []reflect.SelectCase{ { - Dir: reflect.SelectRecv, + Dir: reflect.SelectRecv, Chan: reflect.ValueOf(chCtx.Done()), }, { diff --git a/lib/padreader/padreader.go b/lib/padreader/padreader.go new file mode 100644 index 000000000..135e35b57 --- /dev/null +++ b/lib/padreader/padreader.go @@ -0,0 +1,38 @@ +package padreader + +import ( + "io" + "math/bits" + + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" +) + +func PaddedSize(size uint64) uint64 { + logv := 64 - bits.LeadingZeros64(size) + + sectSize := uint64(1 << logv) + bound := sectorbuilder.GetMaxUserBytesPerStagedSector(sectSize) + if size <= bound { + return bound + } + + return sectorbuilder.GetMaxUserBytesPerStagedSector(1 << (logv + 1)) +} + +type nullReader struct{} + +func (nr nullReader) Read(b []byte) (int, error) { + for i := range b { + b[i] = 0 + } + return len(b), nil +} + +func New(r io.Reader, size uint64) (io.Reader, uint64) { + padSize := PaddedSize(size) + + return io.MultiReader( + io.LimitReader(r, int64(size)), + io.LimitReader(nullReader{}, int64(padSize-size)), + ), padSize +} diff --git a/lib/padreader/padreader_test.go b/lib/padreader/padreader_test.go new file mode 100644 index 000000000..551393b4e --- /dev/null +++ b/lib/padreader/padreader_test.go @@ -0,0 +1,18 @@ +package padreader + +import ( + "gotest.tools/assert" + "testing" +) + +func TestComputePaddedSize(t *testing.T) { + assert.Equal(t, uint64(1040384), PaddedSize(1000000)) + + assert.Equal(t, uint64(1016), PaddedSize(548)) + assert.Equal(t, uint64(1016), PaddedSize(1015)) + assert.Equal(t, uint64(1016), PaddedSize(1016)) + assert.Equal(t, uint64(2032), PaddedSize(1017)) + + assert.Equal(t, uint64(2032), PaddedSize(1024)) + assert.Equal(t, uint64(4064), PaddedSize(2048)) +} diff --git a/lib/sectorbuilder/files.go b/lib/sectorbuilder/files.go new file mode 100644 index 000000000..51b748f50 --- /dev/null +++ b/lib/sectorbuilder/files.go @@ -0,0 +1,88 @@ +package sectorbuilder + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "golang.org/x/xerrors" +) + +func (sb *SectorBuilder) sectorName(sectorID uint64) string { + return fmt.Sprintf("s-%s-%d", sb.Miner, sectorID) +} + +func (sb *SectorBuilder) stagedSectorPath(sectorID uint64) string { + return filepath.Join(sb.stagedDir, sb.sectorName(sectorID)) +} + +func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) { + return os.OpenFile(sb.stagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE, 0644) +} + +func (sb *SectorBuilder) sealedSectorPath(sectorID uint64) (string, error) { + path := filepath.Join(sb.sealedDir, sb.sectorName(sectorID)) + + e, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return "", err + } + + return path, e.Close() +} + +func (sb *SectorBuilder) sectorCacheDir(sectorID uint64) (string, error) { + dir := filepath.Join(sb.cacheDir, sb.sectorName(sectorID)) + + err := os.Mkdir(dir, 0755) + if os.IsExist(err) { + err = nil + } + + return dir, err +} + +func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { + f, ok := r.(*os.File) + if ok { + return f, func() error { return nil }, nil + } + + var w *os.File + + f, w, err := os.Pipe() + if err != nil { + return nil, nil, err + } + + var wait sync.Mutex + var werr error + + wait.Lock() + go func() { + defer wait.Unlock() + + copied, werr := io.CopyN(w, r, n) + if werr != nil { + log.Warnf("toReadableFile: copy error: %+v", werr) + } + + err := w.Close() + if werr == nil && err != nil { + werr = err + log.Warnf("toReadableFile: close error: %+v", err) + return + } + if copied != n { + log.Warnf("copied different amount than expected: %d != %d", copied, n) + werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n) + } + }() + + return f, func() error { + wait.Lock() + return werr + }, nil +} diff --git a/lib/sectorbuilder/mock.go b/lib/sectorbuilder/mock.go new file mode 100644 index 000000000..e9661ada1 --- /dev/null +++ b/lib/sectorbuilder/mock.go @@ -0,0 +1,55 @@ +package sectorbuilder + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/node/modules/dtypes" +) + +func TempSectorbuilder(sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, func(), error) { + dir, err := ioutil.TempDir("", "sbtest") + if err != nil { + return nil, nil, err + } + + sb, err := TempSectorbuilderDir(dir, sectorSize, ds) + return sb, func() { + sb.Destroy() + + if err := os.RemoveAll(dir); err != nil { + log.Warn("failed to clean up temp sectorbuilder: ", err) + } + }, err +} + +func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, error) { + addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq") + if err != nil { + return nil, err + } + + metadata := filepath.Join(dir, "meta") + sealed := filepath.Join(dir, "sealed") + staging := filepath.Join(dir, "staging") + cache := filepath.Join(dir, "cache") + + sb, err := New(&Config{ + SectorSize: sectorSize, + + SealedDir: sealed, + StagedDir: staging, + MetadataDir: metadata, + CacheDir: cache, + + WorkerThreads: 2, + Miner: addr, + }, ds) + if err != nil { + return nil, err + } + + return sb, nil +} diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index a2e4499b7..35f37ac2d 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -1,19 +1,28 @@ package sectorbuilder import ( + "fmt" "io" "os" "sort" + "strconv" "sync" "unsafe" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - + "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) +const PoStReservedWorkers = 1 +const PoRepProofPartitions = 2 + +var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last") + var log = logging.Logger("sectorbuilder") type SectorSealingStatus = sectorbuilder.SectorSealingStatus @@ -26,33 +35,111 @@ type SectorInfo = sectorbuilder.SectorInfo type SealTicket = sectorbuilder.SealTicket -type SealedSectorMetadata = sectorbuilder.SealedSectorMetadata +type SealSeed = sectorbuilder.SealSeed + +type SealPreCommitOutput = sectorbuilder.SealPreCommitOutput + +type SealCommitOutput = sectorbuilder.SealCommitOutput + +type PublicPieceInfo = sectorbuilder.PublicPieceInfo + +type RawSealPreCommitOutput = sectorbuilder.RawSealPreCommitOutput const CommLen = sectorbuilder.CommitmentBytesLen type SectorBuilder struct { handle unsafe.Pointer + ds dtypes.MetadataDS + idLk sync.Mutex + + ssize uint64 + + Miner address.Address + + stagedDir string + sealedDir string + cacheDir string + + rateLimit chan struct{} } -type SectorBuilderConfig struct { - SectorSize uint64 - Miner address.Address +type Config struct { + SectorSize uint64 + Miner address.Address + + WorkerThreads uint8 + + CacheDir string SealedDir string StagedDir string MetadataDir string } -func New(cfg *SectorBuilderConfig) (*SectorBuilder, error) { +func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) { + if cfg.WorkerThreads <= PoStReservedWorkers { + return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers+1, cfg.WorkerThreads) + } + proverId := addressToProverID(cfg.Miner) - sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 1, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, 16) + for _, dir := range []string{cfg.StagedDir, cfg.SealedDir, cfg.CacheDir, cfg.MetadataDir} { + if err := os.Mkdir(dir, 0755); err != nil { + if os.IsExist(err) { + continue + } + return nil, err + } + } + + var lastUsedID uint64 + b, err := ds.Get(lastSectorIdKey) + switch err { + case nil: + i, err := strconv.ParseInt(string(b), 10, 64) + if err != nil { + return nil, err + } + lastUsedID = uint64(i) + case datastore.ErrNotFound: + default: + return nil, err + } + + sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, PoRepProofPartitions, lastUsedID, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads) if err != nil { return nil, err } - return &SectorBuilder{ + sb := &SectorBuilder{ handle: sbp, - }, nil + ds: ds, + + ssize: cfg.SectorSize, + + stagedDir: cfg.StagedDir, + sealedDir: cfg.SealedDir, + cacheDir: cfg.CacheDir, + + Miner: cfg.Miner, + rateLimit: make(chan struct{}, cfg.WorkerThreads-PoStReservedWorkers), + } + + return sb, nil +} + +func (sb *SectorBuilder) rlimit() func() { + if cap(sb.rateLimit) == len(sb.rateLimit) { + log.Warn("rate-limiting sectorbuilder call") + } + sb.rateLimit <- struct{}{} + + return func() { + <-sb.rateLimit + } +} + +func (sb *SectorBuilder) WorkerStats() (free, reserved, total int) { + return cap(sb.rateLimit) - len(sb.rateLimit), PoStReservedWorkers, cap(sb.rateLimit) + PoStReservedWorkers } func addressToProverID(a address.Address) [32]byte { @@ -65,31 +152,161 @@ func (sb *SectorBuilder) Destroy() { sectorbuilder.DestroySectorBuilder(sb.handle) } -func (sb *SectorBuilder) AddPiece(pieceKey string, pieceSize uint64, file io.Reader) (uint64, error) { +func (sb *SectorBuilder) AcquireSectorId() (uint64, error) { + sb.idLk.Lock() + defer sb.idLk.Unlock() + + id, err := sectorbuilder.AcquireSectorId(sb.handle) + if err != nil { + return 0, err + } + err = sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id))) + if err != nil { + return 0, err + } + return id, nil +} + +func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) { f, werr, err := toReadableFile(file, int64(pieceSize)) if err != nil { - return 0, err + return PublicPieceInfo{}, err } - sectorID, err := sectorbuilder.AddPieceFromFile(sb.handle, pieceKey, pieceSize, f) + ret := sb.rlimit() + defer ret() + + stagedFile, err := sb.stagedSectorFile(sectorId) if err != nil { - return 0, err + return PublicPieceInfo{}, err } - return sectorID, werr() + _, _, commP, err := sectorbuilder.StandaloneWriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes) + if err != nil { + return PublicPieceInfo{}, err + } + + if err := stagedFile.Close(); err != nil { + return PublicPieceInfo{}, err + } + + if err := f.Close(); err != nil { + return PublicPieceInfo{}, err + } + + return PublicPieceInfo{ + Size: pieceSize, + CommP: commP, + }, werr() } // TODO: should *really really* return an io.ReadCloser func (sb *SectorBuilder) ReadPieceFromSealedSector(pieceKey string) ([]byte, error) { + ret := sb.rlimit() + defer ret() + return sectorbuilder.ReadPieceFromSealedSector(sb.handle, pieceKey) } -func (sb *SectorBuilder) SealSector(sectorID uint64, ticket SealTicket) (SealedSectorMetadata, error) { - return sectorbuilder.SealSector(sb.handle, sectorID, ticket) +func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, pieces []PublicPieceInfo) (RawSealPreCommitOutput, error) { + ret := sb.rlimit() + defer ret() + + cacheDir, err := sb.sectorCacheDir(sectorID) + if err != nil { + return RawSealPreCommitOutput{}, err + } + + sealedPath, err := sb.sealedSectorPath(sectorID) + if err != nil { + return RawSealPreCommitOutput{}, err + } + + var sum uint64 + for _, piece := range pieces { + sum += piece.Size + } + ussize := UserBytesForSectorSize(sb.ssize) + if sum != ussize { + return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum)) + } + + stagedPath := sb.stagedSectorPath(sectorID) + + rspco, err := sectorbuilder.StandaloneSealPreCommit( + sb.ssize, + PoRepProofPartitions, + cacheDir, + stagedPath, + sealedPath, + sectorID, + addressToProverID(sb.Miner), + ticket.TicketBytes, + pieces, + ) + if err != nil { + return RawSealPreCommitOutput{}, xerrors.Errorf("presealing sector %d (%s): %w", sectorID, stagedPath, err) + } + + return rspco, nil } -func (sb *SectorBuilder) ResumeSealSector(sectorID uint64) (SealedSectorMetadata, error) { - return sectorbuilder.ResumeSealSector(sb.handle, sectorID) +func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, pieceKeys []string, rspco RawSealPreCommitOutput) (proof []byte, err error) { + ret := sb.rlimit() + defer ret() + + cacheDir, err := sb.sectorCacheDir(sectorID) + if err != nil { + return nil, err + } + + proof, err = sectorbuilder.StandaloneSealCommit( + sb.ssize, + PoRepProofPartitions, + cacheDir, + sectorID, + addressToProverID(sb.Miner), + ticket.TicketBytes, + seed.TicketBytes, + pieces, + rspco, + ) + if err != nil { + return nil, xerrors.Errorf("StandaloneSealCommit: %w", err) + } + + pmeta := make([]sectorbuilder.PieceMetadata, len(pieces)) + for i, piece := range pieces { + pmeta[i] = sectorbuilder.PieceMetadata{ + Key: pieceKeys[i], + Size: piece.Size, + CommP: piece.CommP, + } + } + + sealedPath, err := sb.sealedSectorPath(sectorID) + if err != nil { + return nil, err + } + + err = sectorbuilder.ImportSealedSector( + sb.handle, + sectorID, + cacheDir, + sealedPath, + ticket, + seed, + rspco.CommR, + rspco.CommD, + rspco.CommC, + rspco.CommRLast, + proof, + pmeta, + ) + if err != nil { + return nil, xerrors.Errorf("ImportSealedSector: %w", err) + } + return proof, nil } func (sb *SectorBuilder) SealStatus(sector uint64) (SectorSealingStatus, error) { @@ -120,24 +337,21 @@ func (sb *SectorBuilder) GeneratePoSt(sectorInfo SortedSectorInfo, challengeSeed return sectorbuilder.GeneratePoSt(sb.handle, sectorInfo, challengeSeed, faults) } +func (sb *SectorBuilder) SectorSize() uint64 { + return sb.ssize +} + var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector -func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, sectorID uint64, proof []byte) (bool, error) { - var commRa, commDa, ticketa [32]byte +func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) { + var commRa, commDa, ticketa, seeda [32]byte copy(commRa[:], commR) copy(commDa[:], commD) copy(ticketa[:], ticket) + copy(seeda[:], seed) proverIDa := addressToProverID(proverID) - return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, sectorID, proof) -} - -func VerifyPieceInclusionProof(sectorSize uint64, pieceSize uint64, commP []byte, commD []byte, proof []byte) (bool, error) { - var commPa, commDa [32]byte - copy(commPa[:], commP) - copy(commDa[:], commD) - - return sectorbuilder.VerifyPieceInclusionProof(sectorSize, pieceSize, commPa, commDa, proof) + return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof) } func NewSortedSectorInfo(sectors []SectorInfo) SortedSectorInfo { @@ -162,36 +376,6 @@ func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen] return commP, werr() } -func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { - f, ok := r.(*os.File) - if ok { - return f, func() error { return nil }, nil - } - - var w *os.File - - f, w, err := os.Pipe() - if err != nil { - return nil, nil, err - } - - var wait sync.Mutex - var werr error - - wait.Lock() - go func() { - defer wait.Unlock() - - _, werr = io.CopyN(w, r, n) - - err := w.Close() - if werr == nil { - werr = err - } - }() - - return f, func() error { - wait.Lock() - return werr - }, nil +func GenerateDataCommitment(ssize uint64, pieces []PublicPieceInfo) ([CommLen]byte, error) { + return sectorbuilder.GenerateDataCommitment(ssize, pieces) } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index b40a0b9cd..e02368913 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -1,86 +1,137 @@ package sectorbuilder_test import ( - "context" "io" "io/ioutil" "math/rand" - "path/filepath" + "os" "testing" "github.com/ipfs/go-datastore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/storage/sector" ) const sectorSize = 1024 func TestSealAndVerify(t *testing.T) { - t.Skip("this is slow") + //t.Skip("this is slow") + os.Setenv("BELLMAN_NO_GPU", "1") + build.SectorSizes = []uint64{sectorSize} if err := build.GetParams(true); err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } + sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize, datastore.NewMapDatastore()) + if err != nil { + t.Fatalf("%+v", err) + } + defer cleanup() + + dlen := sectorbuilder.UserBytesForSectorSize(sectorSize) + + sid, err := sb.AcquireSectorId() + if err != nil { + t.Fatalf("%+v", err) + } + + r := io.LimitReader(rand.New(rand.NewSource(42)), int64(dlen)) + ppi, err := sb.AddPiece(dlen, sid, r, []uint64{}) + if err != nil { + t.Fatalf("%+v", err) + } + + ticket := sectorbuilder.SealTicket{ + BlockHeight: 5, + TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2}, + } + + pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{ppi}) + if err != nil { + t.Fatalf("%+v", err) + } + + seed := sectorbuilder.SealSeed{ + BlockHeight: 15, + TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}, + } + + proof, err := sb.SealCommit(sid, ticket, seed, []sectorbuilder.PublicPieceInfo{ppi}, []string{"foo"}, pco) + if err != nil { + t.Fatalf("%+v", err) + } + + ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], pco.CommD[:], sb.Miner, ticket.TicketBytes[:], seed.TicketBytes[:], sid, proof) + if err != nil { + t.Fatalf("%+v", err) + } + + if !ok { + t.Fatal("proof failed to validate") + } + + cSeed := [32]byte{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9} + + ssi := sectorbuilder.NewSortedSectorInfo([]sectorbuilder.SectorInfo{{ + SectorID: sid, + CommR: pco.CommR, + }}) + + postProof, err := sb.GeneratePoSt(ssi, cSeed, []uint64{}) + if err != nil { + t.Fatalf("%+v", err) + } + + ok, err = sectorbuilder.VerifyPost(sb.SectorSize(), ssi, cSeed, postProof, []uint64{}) + if err != nil { + t.Fatalf("%+v", err) + } + if !ok { + t.Fatal("bad post") + } +} + +func TestAcquireID(t *testing.T) { + ds := datastore.NewMapDatastore() + dir, err := ioutil.TempDir("", "sbtest") if err != nil { t.Fatal(err) } - addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq") + sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) if err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } - metadata := filepath.Join(dir, "meta") - sealed := filepath.Join(dir, "sealed") - staging := filepath.Join(dir, "staging") + assertAcquire := func(expect uint64) { + id, err := sb.AcquireSectorId() + require.NoError(t, err) + assert.Equal(t, expect, id) + } - sb, err := sectorbuilder.New(§orbuilder.SectorBuilderConfig{ - SectorSize: sectorSize, - SealedDir: sealed, - StagedDir: staging, - MetadataDir: metadata, - Miner: addr, - }) + assertAcquire(1) + assertAcquire(2) + assertAcquire(3) + + sb.Destroy() + + sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) if err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } - // TODO: Consider fixing - store := sector.NewStore(sb, datastore.NewMapDatastore(), func(ctx context.Context) (*sectorbuilder.SealTicket, error) { - return §orbuilder.SealTicket{ - BlockHeight: 5, - TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2}, - }, nil - }) + assertAcquire(4) + assertAcquire(5) + assertAcquire(6) - store.Service() - - dlen := sectorbuilder.UserBytesForSectorSize(sectorSize) - - r := io.LimitReader(rand.New(rand.NewSource(42)), int64(dlen)) - sid, err := store.AddPiece("foo", dlen, r) - if err != nil { - t.Fatal(err) - } - - if err := store.SealSector(context.TODO(), sid); err != nil { - t.Fatal(err) - } - - ssinfo := <-store.Incoming() - - ok, err := sectorbuilder.VerifySeal(sectorSize, ssinfo.CommR[:], ssinfo.CommD[:], addr, ssinfo.Ticket.TicketBytes[:], ssinfo.SectorID, ssinfo.Proof) - if err != nil { - t.Fatal(err) - } - - if !ok { - t.Fatal("proof failed to validate") + sb.Destroy() + if err := os.RemoveAll(dir); err != nil { + t.Error(err) } } diff --git a/lib/statestore/store.go b/lib/statestore/store.go new file mode 100644 index 000000000..994938223 --- /dev/null +++ b/lib/statestore/store.go @@ -0,0 +1,161 @@ +package statestore + +import ( + "bytes" + "fmt" + "reflect" + + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/lib/cborutil" +) + +type StateStore struct { + ds datastore.Datastore +} + +func New(ds datastore.Datastore) *StateStore { + return &StateStore{ds: ds} +} + +func toKey(k interface{}) datastore.Key { + switch t := k.(type) { + case uint64: + return datastore.NewKey(fmt.Sprint(t)) + case fmt.Stringer: + return datastore.NewKey(t.String()) + default: + panic("unexpected key type") + } +} + +func (st *StateStore) Begin(i interface{}, state interface{}) error { + k := toKey(i) + has, err := st.ds.Has(k) + if err != nil { + return err + } + if has { + return xerrors.Errorf("already tracking state for %v", i) + } + + b, err := cborutil.Dump(state) + if err != nil { + return err + } + + return st.ds.Put(k, b) +} + +func (st *StateStore) End(i interface{}) error { + k := toKey(i) + has, err := st.ds.Has(k) + if err != nil { + return err + } + if !has { + return xerrors.Errorf("No state for %s", i) + } + return st.ds.Delete(k) +} + +func cborMutator(mutator interface{}) func([]byte) ([]byte, error) { + rmut := reflect.ValueOf(mutator) + + return func(in []byte) ([]byte, error) { + state := reflect.New(rmut.Type().In(0).Elem()) + + err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface()) + if err != nil { + return nil, err + } + + out := rmut.Call([]reflect.Value{state}) + + if err := out[0].Interface(); err != nil { + return nil, err.(error) + } + + return cborutil.Dump(state.Interface()) + } +} + +// mutator func(*T) error +func (st *StateStore) Mutate(i interface{}, mutator interface{}) error { + return st.mutate(i, cborMutator(mutator)) +} + +func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)) error { + k := toKey(i) + has, err := st.ds.Has(k) + if err != nil { + return err + } + if !has { + return xerrors.Errorf("No state for %s", i) + } + + cur, err := st.ds.Get(k) + if err != nil { + return err + } + + mutated, err := mutator(cur) + if err != nil { + return err + } + + return st.ds.Put(k, mutated) +} + +func (st *StateStore) Has(i interface{}) (bool, error) { + return st.ds.Has(toKey(i)) +} + +func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error { + k := toKey(i) + val, err := st.ds.Get(k) + if err != nil { + if xerrors.Is(err, datastore.ErrNotFound) { + return xerrors.Errorf("No state for %s: %w", i, err) + } + return err + } + + return out.UnmarshalCBOR(bytes.NewReader(val)) +} + +// out: *[]T +func (st *StateStore) List(out interface{}) error { + res, err := st.ds.Query(query.Query{}) + if err != nil { + return err + } + defer res.Close() + + outT := reflect.TypeOf(out).Elem().Elem() + rout := reflect.ValueOf(out) + + for { + res, ok := res.NextSync() + if !ok { + break + } + if res.Error != nil { + return res.Error + } + + elem := reflect.New(outT) + err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface()) + if err != nil { + return err + } + + rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem())) + } + + return nil +} diff --git a/lib/statestore/store_test.go b/lib/statestore/store_test.go new file mode 100644 index 000000000..91724bbf8 --- /dev/null +++ b/lib/statestore/store_test.go @@ -0,0 +1,38 @@ +package statestore + +import ( + "testing" + + "github.com/ipfs/go-datastore" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/cborutil" +) + +func TestList(t *testing.T) { + ds := datastore.NewMapDatastore() + + e, err := cborutil.Dump(types.NewInt(7)) + if err != nil { + t.Fatal(err) + } + + if err := ds.Put(datastore.NewKey("/2"), e); err != nil { + t.Fatal(err) + } + + st := &StateStore{ds: ds} + + var out []types.BigInt + if err := st.List(&out); err != nil { + t.Fatal(err) + } + + if len(out) != 1 { + t.Fatal("wrong len") + } + + if out[0].Int64() != 7 { + t.Fatal("wrong data") + } +} diff --git a/lotuspond/front/src/Address.js b/lotuspond/front/src/Address.js index db6870e9a..1ac98d7f2 100644 --- a/lotuspond/front/src/Address.js +++ b/lotuspond/front/src/Address.js @@ -133,7 +133,7 @@ class Address extends React.Component { let transfer = if(this.props.transfer) { - transfer =  {this.props.transfer}FIL + transfer =  {this.props.transfer}FIL } let minerInfo = diff --git a/lotuspond/front/src/Client.js b/lotuspond/front/src/Client.js index 37606223b..70aabfd6f 100644 --- a/lotuspond/front/src/Client.js +++ b/lotuspond/front/src/Client.js @@ -21,7 +21,7 @@ class Client extends React.Component { this.state = { miners: ["t0101"], - ask: {Price: "500000000"}, + ask: {Price: "1000000000"}, // 2x min default ask to account for bin packing (could also do the math correctly below, but..) kbs: 1, blocks: 12, @@ -52,7 +52,7 @@ class Client extends React.Component { update = (name) => (e) => this.setState({ [name]: e.target.value }); makeDeal = async () => { - let perBlk = this.state.ask.Price * this.state.kbs * 1000 / (1 << 30) + let perBlk = this.state.ask.Price * this.state.kbs * 1000 / (1 << 30) * 2 let file = await this.props.pondClient.call('Pond.CreateRandomFile', [this.state.kbs * 1000]) // 1024 won't fit in 1k blocks :( let cid = await this.props.client.call('Filecoin.ClientImport', [file]) diff --git a/lotuspond/front/src/StorageNode.js b/lotuspond/front/src/StorageNode.js index 741932db9..c07abc590 100644 --- a/lotuspond/front/src/StorageNode.js +++ b/lotuspond/front/src/StorageNode.js @@ -8,13 +8,14 @@ const stateConnecting = 'connecting' const stateGettingToken = 'getting-token' let sealCodes = [ - "Unknown", - "Pending", - "Failed", - "Sealing", - "Sealed", - "Paused", - "ReadyForSealing", + "Undefined", + "Empty", + "Packing", + "Unsealed", + "PreCommitting", + "PreCommitted", + "Committing", + "Proving", ] class StorageNode extends React.Component { diff --git a/lotuspond/front/src/chain/methods.js b/lotuspond/front/src/chain/methods.js index b9e153ef7..172531c5a 100644 --- a/lotuspond/front/src/chain/methods.js +++ b/lotuspond/front/src/chain/methods.js @@ -34,8 +34,9 @@ export default { [code.miner]: [ "Send", "Constructor", - "CommitSector", - "SubmitPost", + "PreCommitSector", + "ProveCommitSector", + "SubmitPoSt", "SlashStorageFault", "GetCurrentProvingSet", "ArbitrateDeal", @@ -49,8 +50,8 @@ export default { "ChangeWorker", "IsSlashed", "IsLate", - "PaymentVerifyInclusion", - "PaymentVerifySector", + "DeclareFaults", + "SlashConsensusFault", ], [code.multisig]: [ "Send", diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index c79d736f7..eac9ea418 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -49,7 +49,7 @@ func (api *api) Spawn() (nodeInfo, error) { cmd := exec.Command("./lotus", "daemon", "--bootstrap=false", genParam, "--api", fmt.Sprintf("%d", 2500+id)) cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw) cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw) - cmd.Env = []string{"LOTUS_PATH=" + dir} + cmd.Env = append(os.Environ(), "LOTUS_PATH="+dir) if err := cmd.Start(); err != nil { return nodeInfo{}, err } @@ -112,7 +112,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { cmd := exec.Command("./lotus-storage-miner", initArgs...) cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile) cmd.Stdout = io.MultiWriter(os.Stdout, logfile) - cmd.Env = []string{"LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo} + cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH="+dir, "LOTUS_PATH="+fullNodeRepo) if err := cmd.Run(); err != nil { return nodeInfo{}, err } @@ -124,7 +124,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { cmd = exec.Command("./lotus-storage-miner", "run", "--api", fmt.Sprintf("%d", 2500+id)) cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw) cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw) - cmd.Env = []string{"LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo} + cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH="+dir, "LOTUS_PATH="+fullNodeRepo) if err := cmd.Start(); err != nil { return nodeInfo{}, err } diff --git a/miner/miner.go b/miner/miner.go index 9f7c44478..72ec090c7 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -230,7 +230,7 @@ func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error) } func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, error) { - log.Infow("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids())) + log.Debugw("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids())) ticket, err := m.scratchTicket(ctx, base) if err != nil { return nil, errors.Wrap(err, "scratching ticket failed") diff --git a/node/builder.go b/node/builder.go index d082ea635..183472dbc 100644 --- a/node/builder.go +++ b/node/builder.go @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/deals" + "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" @@ -41,8 +42,6 @@ import ( "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/commitment" - "github.com/filecoin-project/lotus/storage/sector" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -223,17 +222,16 @@ func Online() Option { Override(new(*paych.Store), paych.NewStore), Override(new(*paych.Manager), paych.NewManager), + Override(new(*market.FundMgr), market.NewFundMgr), Override(new(*miner.Miner), miner.NewMiner), ), // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.RepoStorageMiner }, - Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New), - Override(new(*sector.Store), sector.NewStore), + Override(new(*sectorbuilder.SectorBuilder), modules.SectorBuilder), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), - Override(new(*commitment.Tracker), commitment.NewTracker), - Override(new(sector.TicketFn), modules.SealTicketGen), + Override(new(storage.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), Override(new(dtypes.StagingDAG), modules.StagingDAG), @@ -242,7 +240,6 @@ func Online() Option { Override(new(*deals.Provider), deals.NewProvider), Override(HandleRetrievalKey, modules.HandleRetrieval), Override(HandleDealsKey, modules.HandleDeals), - Override(RunSectorServiceKey, modules.RunSectorService), Override(RegisterMinerKey, modules.RegisterMiner), ), ) diff --git a/node/hello/hello.go b/node/hello/hello.go index d512d1d27..90af18923 100644 --- a/node/hello/hello.go +++ b/node/hello/hello.go @@ -3,6 +3,7 @@ package hello import ( "context" "fmt" + "go.uber.org/fx" "github.com/ipfs/go-cid" @@ -15,7 +16,7 @@ import ( "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/peermgr" ) @@ -65,7 +66,7 @@ func (hs *Service) HandleStream(s inet.Stream) { defer s.Close() var hmsg Message - if err := cborrpc.ReadCborRPC(s, &hmsg); err != nil { + if err := cborutil.ReadCborRPC(s, &hmsg); err != nil { log.Infow("failed to read hello message", "error", err) return } @@ -119,7 +120,7 @@ func (hs *Service) SayHello(ctx context.Context, pid peer.ID) error { fmt.Println("SENDING HELLO MESSAGE: ", hts.Cids(), hts.Height()) fmt.Println("hello message genesis: ", gen.Cid()) - if err := cborrpc.WriteCborRPC(s, hmsg); err != nil { + if err := cborutil.WriteCborRPC(s, hmsg); err != nil { return err } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a785bef44..8f31df6cd 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -3,11 +3,12 @@ package client import ( "context" "errors" - "golang.org/x/xerrors" "io" "math" "os" + "golang.org/x/xerrors" + "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" "github.com/ipfs/go-filestore" @@ -23,7 +24,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/store" @@ -57,23 +57,17 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A // TODO: make this a param self, err := a.WalletDefaultAddress(ctx) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to get default address: %w", err) } - // get miner peerID - msg := &types.Message{ - To: miner, - From: miner, - Method: actors.MAMethods.GetPeerID, + pid, err := a.StateMinerPeerID(ctx, miner, nil) + if err != nil { + return nil, xerrors.Errorf("failed getting peer ID: %w", err) } - r, err := a.StateCall(ctx, msg, nil) + mw, err := a.StateMinerWorker(ctx, miner, nil) if err != nil { - return nil, err - } - pid, err := peer.IDFromBytes(r.Return) - if err != nil { - return nil, err + return nil, xerrors.Errorf("failed getting miner worker: %w", err) } proposal := deals.ClientDealProposal{ @@ -81,14 +75,18 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A PricePerEpoch: epochPrice, ProposalExpiration: math.MaxUint64, // TODO: set something reasonable Duration: blocksDuration, - ProviderAddress: miner, Client: self, + ProviderAddress: miner, + MinerWorker: mw, MinerID: pid, } c, err := a.DealClient.Start(ctx, proposal) - // TODO: send updated voucher with PaymentVerifySector for cheaper validation (validate the sector the miner sent us first!) - return &c, err + if err != nil { + return nil, xerrors.Errorf("failed to start deal: %w", err) + } + + return &c, nil } func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { @@ -115,6 +113,22 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { return out, nil } +func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, error) { + v, err := a.DealClient.GetDeal(d) + if err != nil { + return nil, err + } + return &api.DealInfo{ + ProposalCid: v.ProposalCid, + State: v.State, + Provider: v.Proposal.Provider, + PieceRef: v.Proposal.PieceRef, + Size: v.Proposal.PieceSize, + PricePerEpoch: v.Proposal.StoragePricePerEpoch, + Duration: v.Proposal.Duration, + }, nil +} + func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) { // TODO: check if we have the ENTIRE dag @@ -177,7 +191,11 @@ func (a *API) ClientImport(ctx context.Context, path string) (cid.Cid, error) { return cid.Undef, err } - return nd.Cid(), bufferedDS.Commit() + if err := bufferedDS.Commit(); err != nil { + return cid.Undef, err + } + + return nd.Cid(), nil } func (a *API) ClientImportLocal(ctx context.Context, f io.Reader) (cid.Cid, error) { diff --git a/node/impl/full.go b/node/impl/full.go index 41c0730bf..76cdc5de4 100644 --- a/node/impl/full.go +++ b/node/impl/full.go @@ -3,11 +3,12 @@ package impl import ( "context" - "github.com/filecoin-project/lotus/node/impl/client" - "github.com/filecoin-project/lotus/node/impl/paych" - logging "github.com/ipfs/go-log" + "github.com/filecoin-project/lotus/node/impl/client" + "github.com/filecoin-project/lotus/node/impl/market" + "github.com/filecoin-project/lotus/node/impl/paych" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/miner" @@ -21,6 +22,7 @@ type FullNodeAPI struct { full.ChainAPI client.API full.MpoolAPI + market.MarketAPI paych.PaychAPI full.StateAPI full.WalletAPI diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 68a134246..93bc973c1 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -3,9 +3,10 @@ package full import ( "bytes" "context" - "github.com/filecoin-project/go-amt-ipld" "strconv" + "github.com/filecoin-project/go-amt-ipld" + cid "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" "github.com/libp2p/go-libp2p-core/peer" @@ -37,11 +38,11 @@ type StateAPI struct { Chain *store.ChainStore } -func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.SectorInfo, error) { +func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) { return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, addr) } -func (a *StateAPI) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.SectorInfo, error) { +func (a *StateAPI) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) { return stmgr.GetMinerProvingSet(ctx, a.StateManager, ts, addr) } @@ -279,3 +280,7 @@ func (a *StateAPI) StateMarketDeals(ctx context.Context, ts *types.TipSet) (map[ } return out, nil } + +func (a *StateAPI) StateMarketStorageDeal(ctx context.Context, dealId uint64, ts *types.TipSet) (*actors.OnChainDeal, error) { + return stmgr.GetStorageDeal(ctx, a.StateManager, dealId, ts) +} diff --git a/node/impl/market/market.go b/node/impl/market/market.go new file mode 100644 index 000000000..3e7724e43 --- /dev/null +++ b/node/impl/market/market.go @@ -0,0 +1,21 @@ +package market + +import ( + "context" + + "go.uber.org/fx" + + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/market" + "github.com/filecoin-project/lotus/chain/types" +) + +type MarketAPI struct { + fx.In + + FMgr *market.FundMgr +} + +func (a *MarketAPI) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error { + return a.FMgr.EnsureAvailable(ctx, addr, amt) +} diff --git a/node/impl/paych/paych.go b/node/impl/paych/paych.go index b89980a00..e4ae9e23f 100644 --- a/node/impl/paych/paych.go +++ b/node/impl/paych/paych.go @@ -162,12 +162,12 @@ func (a *PaychAPI) PaychVoucherCreate(ctx context.Context, pch address.Address, func (a *PaychAPI) paychVoucherCreate(ctx context.Context, pch address.Address, voucher types.SignedVoucher) (*types.SignedVoucher, error) { ci, err := a.PaychMgr.GetChannelInfo(pch) if err != nil { - return nil, err + return nil, xerrors.Errorf("get channel info: %w", err) } nonce, err := a.PaychMgr.NextNonceForLane(ctx, pch, voucher.Lane) if err != nil { - return nil, err + return nil, xerrors.Errorf("getting next nonce for lane: %w", err) } sv := &voucher diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 8eb363fe2..f339fd66a 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -2,31 +2,32 @@ package impl import ( "context" - "fmt" - "io" - "math/rand" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/commitment" - "github.com/filecoin-project/lotus/storage/sector" "github.com/filecoin-project/lotus/storage/sectorblocks" - - "golang.org/x/xerrors" ) type StorageMinerAPI struct { CommonAPI - SectorBuilderConfig *sectorbuilder.SectorBuilderConfig + SectorBuilderConfig *sectorbuilder.Config SectorBuilder *sectorbuilder.SectorBuilder - Sectors *sector.Store SectorBlocks *sectorblocks.SectorBlocks - CommitmentTracker *commitment.Tracker Miner *storage.Miner + Full api.FullNode +} + +func (sm *StorageMinerAPI) WorkerStats(context.Context) (api.WorkerStats, error) { + free, reserved, total := sm.SectorBuilder.WorkerStats() + return api.WorkerStats{ + Free: free, + Reserved: reserved, + Total: total, + }, nil } func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error) { @@ -34,37 +35,44 @@ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error } func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) error { - ssize, err := sm.Miner.SectorSize(ctx) - if err != nil { - return xerrors.Errorf("failed to get miner sector size: %w", err) - } - go func() { - size := sectorbuilder.UserBytesForSectorSize(ssize) - - // TODO: create a deal - name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) - sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size))) - if err != nil { - log.Error(err) - return - } - - if err := sm.Sectors.SealSector(ctx, sectorId); err != nil { - log.Error(err) - return - } - }() - - return err + return sm.Miner.StoreGarbageData() } -func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (sectorbuilder.SectorSealingStatus, error) { - return sm.SectorBuilder.SealStatus(sid) +func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (api.SectorInfo, error) { + info, err := sm.Miner.GetSectorInfo(sid) + if err != nil { + return api.SectorInfo{}, err + } + + deals := make([]uint64, len(info.Pieces)) + for i, piece := range info.Pieces { + deals[i] = piece.DealID + } + + return api.SectorInfo{ + SectorID: sid, + State: info.State, + CommD: info.CommD, + CommR: info.CommR, + Proof: info.Proof, + Deals: deals, + Ticket: info.Ticket.SB(), + Seed: info.Seed.SB(), + }, nil } // List all staged sectors func (sm *StorageMinerAPI) SectorsList(context.Context) ([]uint64, error) { - return sm.SectorBuilder.GetAllStagedSectors() + sectors, err := sm.Miner.ListSectors() + if err != nil { + return nil, err + } + + out := make([]uint64, len(sectors)) + for i, sector := range sectors { + out[i] = sector.SectorID + } + return out, nil } func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.SealedRef, error) { @@ -83,8 +91,4 @@ func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.Sealed return out, nil } -func (sm *StorageMinerAPI) CommitmentsList(ctx context.Context) ([]api.SectorCommitment, error) { - return sm.CommitmentTracker.List() -} - var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/modules/services.go b/node/modules/services.go index 3c38f4461..c5c3df125 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -15,7 +15,6 @@ import ( "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" "github.com/filecoin-project/lotus/retrieval/discovery" - "github.com/filecoin-project/lotus/storage/sector" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { @@ -79,19 +78,6 @@ func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c *deals.Client) { }) } -func RunSectorService(lc fx.Lifecycle, secst *sector.Store) { - lc.Append(fx.Hook{ - OnStart: func(context.Context) error { - secst.Service() - return nil - }, - OnStop: func(context.Context) error { - secst.Stop() - return nil - }, - }) -} - func RetrievalResolver(l *discovery.Local) discovery.PeerResolver { return discovery.Multi(l) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 49a8d1dd5..0c97710cb 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -3,6 +3,7 @@ package modules import ( "context" "fmt" + "math" "path/filepath" "github.com/ipfs/go-bitswap" @@ -27,8 +28,6 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/commitment" - "github.com/filecoin-project/lotus/storage/sector" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -40,8 +39,8 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { return address.NewFromBytes(maddrb) } -func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) { - return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) { +func SectorBuilderConfig(storagePath string, threads uint) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.Config, error) { + return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.Config, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { return nil, err @@ -57,13 +56,21 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNod return nil, err } + if threads > math.MaxUint8 { + return nil, xerrors.Errorf("too many sectorbuilder threads specified: %d, max allowed: %d", threads, math.MaxUint8) + } + + cache := filepath.Join(sp, "cache") metadata := filepath.Join(sp, "meta") sealed := filepath.Join(sp, "sealed") staging := filepath.Join(sp, "staging") - sb := §orbuilder.SectorBuilderConfig{ - Miner: minerAddr, - SectorSize: ssize, + sb := §orbuilder.Config{ + Miner: minerAddr, + SectorSize: ssize, + WorkerThreads: uint8(threads), + + CacheDir: cache, MetadataDir: metadata, SealedDir: sealed, StagedDir: staging, @@ -73,13 +80,13 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNod } } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, secst *sector.Store, commt *commitment.Tracker) (*storage.Miner, error) { +func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } - sm, err := storage.NewMiner(api, maddr, h, ds, secst, commt) + sm, err := storage.NewMiner(api, maddr, h, ds, sb, tktFn) if err != nil { return nil, err } @@ -90,6 +97,7 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h OnStart: func(context.Context) error { return sm.Run(ctx) }, + OnStop: sm.Stop, }) return sm, nil @@ -168,7 +176,23 @@ func RegisterMiner(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode) erro return nil } -func SealTicketGen(api api.FullNode) sector.TicketFn { +func SectorBuilder(lc fx.Lifecycle, cfg *sectorbuilder.Config, ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilder, error) { + sb, err := sectorbuilder.New(cfg, ds) + if err != nil { + return nil, err + } + + lc.Append(fx.Hook{ + OnStop: func(context.Context) error { + sb.Destroy() + return nil + }, + }) + + return sb, nil +} + +func SealTicketGen(api api.FullNode) storage.TicketFn { return func(ctx context.Context) (*sectorbuilder.SealTicket, error) { ts, err := api.ChainHead(ctx) if err != nil { @@ -186,7 +210,7 @@ func SealTicketGen(api api.FullNode) sector.TicketFn { } return §orbuilder.SealTicket{ - BlockHeight: ts.Height() - build.SealRandomnessLookback, + BlockHeight: ts.Height(), TicketBytes: tkt, }, nil } diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index b53e33319..6c81f68ba 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -25,7 +25,7 @@ import ( var glog = logging.Logger("genesis") -func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { +func MakeGenesisMem(out io.Writer, minerPid peer.ID) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") @@ -38,7 +38,7 @@ func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wal gmc := &gen.GenMinerCfg{ Owners: []address.Address{w}, Workers: []address.Address{w}, - PeerIDs: []peer.ID{"peerID 1"}, + PeerIDs: []peer.ID{minerPid}, } alloc := map[address.Address]types.BigInt{ w: types.FromFil(10000), diff --git a/node/node_test.go b/node/node_test.go index 82ab996bd..1d8c20b03 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -31,15 +31,12 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) -func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, tnd test.TestNode) test.TestStorageNode { +func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode { r := repo.NewMemory(nil) lr, err := r.Lock(repo.RepoStorageMiner) require.NoError(t, err) - pk, _, err := crypto.GenerateEd25519Key(rand.Reader) - require.NoError(t, err) - ks, err := lr.KeyStore() require.NoError(t, err) @@ -93,7 +90,9 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a node.Repo(r), node.Test(), - node.Override(new(*sectorbuilder.SectorBuilderConfig), modules.SectorBuilderConfig(secbpath)), + node.MockHost(mn), + + node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(secbpath, 2)), node.Override(new(api.FullNode), tnd), ) require.NoError(t, err) @@ -115,12 +114,18 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te fulls := make([]test.TestNode, nFull) storers := make([]test.TestStorageNode, len(storage)) + pk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + minerPid, err := peer.IDFromPrivateKey(pk) + require.NoError(t, err) + var genbuf bytes.Buffer for i := 0; i < nFull; i++ { var genesis node.Option if i == 0 { - genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf)) + genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, minerPid)) } else { genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) } @@ -171,7 +176,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te genMiner, err := address.NewFromString("t0101") require.NoError(t, err) - storers[i] = testStorageNode(ctx, t, wa, genMiner, f) + storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn) } if err := mn.LinkAll(); err != nil { @@ -221,3 +226,7 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test func TestAPIRPC(t *testing.T) { test.TestApis(t, rpcBuilder) } + +func TestAPIDealFlow(t *testing.T) { + test.TestDealFlow(t, builder) +} diff --git a/node/options.go b/node/options.go index 58576a950..9c8a79710 100644 --- a/node/options.go +++ b/node/options.go @@ -1,8 +1,9 @@ package node import ( - "go.uber.org/fx" "reflect" + + "go.uber.org/fx" ) // Option is a functional option which can be used with the New function to diff --git a/paych/cbor_gen.go b/paych/cbor_gen.go new file mode 100644 index 000000000..e640f3057 --- /dev/null +++ b/paych/cbor_gen.go @@ -0,0 +1,231 @@ +package paych + +import ( + "fmt" + "io" + + "github.com/filecoin-project/lotus/chain/types" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *VoucherInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Voucher (types.SignedVoucher) (struct) + if err := t.Voucher.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Proof ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { + return err + } + if _, err := w.Write(t.Proof); err != nil { + return err + } + return nil +} + +func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Voucher (types.SignedVoucher) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Voucher = new(types.SignedVoucher) + if err := t.Voucher.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + // t.t.Proof ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Proof: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Proof = make([]byte, extra) + if _, err := io.ReadFull(br, t.Proof); err != nil { + return err + } + return nil +} + +func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{134}); err != nil { + return err + } + + // t.t.Channel (address.Address) (struct) + if err := t.Channel.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Control (address.Address) (struct) + if err := t.Control.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Target (address.Address) (struct) + if err := t.Target.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Direction (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Direction))); err != nil { + return err + } + + // t.t.Vouchers ([]*paych.VoucherInfo) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil { + return err + } + for _, v := range t.Vouchers { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.t.NextLane (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextLane))); err != nil { + return err + } + return nil +} + +func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 6 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Channel (address.Address) (struct) + + { + + if err := t.Channel.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Control (address.Address) (struct) + + { + + if err := t.Control.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Target (address.Address) (struct) + + { + + if err := t.Target.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Direction (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Direction = uint64(extra) + // t.t.Vouchers ([]*paych.VoucherInfo) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Vouchers: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Vouchers = make([]*VoucherInfo, extra) + } + for i := 0; i < int(extra); i++ { + + var v VoucherInfo + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Vouchers[i] = &v + } + + // t.t.NextLane (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.NextLane = uint64(extra) + return nil +} diff --git a/paych/store.go b/paych/store.go index cc78b4b5b..f08969630 100644 --- a/paych/store.go +++ b/paych/store.go @@ -1,6 +1,7 @@ package paych import ( + "bytes" "errors" "fmt" "strings" @@ -9,21 +10,16 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" dsq "github.com/ipfs/go-datastore/query" - cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + cborrpc "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/node/modules/dtypes" ) var ErrChannelNotTracked = errors.New("channel not tracked") -func init() { - cbor.RegisterCborType(VoucherInfo{}) - cbor.RegisterCborType(ChannelInfo{}) -} - type Store struct { lk sync.Mutex // TODO: this can be split per paych @@ -52,7 +48,7 @@ type ChannelInfo struct { Control address.Address Target address.Address - Direction int + Direction uint64 Vouchers []*VoucherInfo NextLane uint64 } @@ -64,7 +60,7 @@ func dskeyForChannel(addr address.Address) datastore.Key { func (ps *Store) putChannelInfo(ci *ChannelInfo) error { k := dskeyForChannel(ci.Channel) - b, err := cbor.DumpObject(ci) + b, err := cborrpc.Dump(ci) if err != nil { return err } @@ -84,7 +80,7 @@ func (ps *Store) getChannelInfo(addr address.Address) (*ChannelInfo, error) { } var ci ChannelInfo - if err := cbor.DecodeInto(b, &ci); err != nil { + if err := ci.UnmarshalCBOR(bytes.NewReader(b)); err != nil { return nil, err } @@ -161,7 +157,7 @@ func (ps *Store) findChan(filter func(*ChannelInfo) bool) (address.Address, erro return address.Undef, err } - if err := cbor.DecodeInto(res.Value, &ci); err != nil { + if err := ci.UnmarshalCBOR(bytes.NewReader(res.Value)); err != nil { return address.Undef, err } diff --git a/peermgr/peermgr.go b/peermgr/peermgr.go index d8a0bea77..4ad64a5a1 100644 --- a/peermgr/peermgr.go +++ b/peermgr/peermgr.go @@ -2,10 +2,11 @@ package peermgr import ( "context" - "github.com/filecoin-project/lotus/node/modules/dtypes" "sync" "time" + "github.com/filecoin-project/lotus/node/modules/dtypes" + host "github.com/libp2p/go-libp2p-core/host" net "github.com/libp2p/go-libp2p-core/network" peer "github.com/libp2p/go-libp2p-core/peer" diff --git a/retrieval/cbor_gen.go b/retrieval/cbor_gen.go new file mode 100644 index 000000000..3feb15dd1 --- /dev/null +++ b/retrieval/cbor_gen.go @@ -0,0 +1,464 @@ +package retrieval + +import ( + "fmt" + "io" + + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *RetParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Unixfs0 (retrieval.Unixfs0Offer) (struct) + if err := t.Unixfs0.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *RetParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Unixfs0 (retrieval.Unixfs0Offer) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Unixfs0 = new(Unixfs0Offer) + if err := t.Unixfs0.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + return nil +} + +func (t *Query) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Piece (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Piece); err != nil { + return xerrors.Errorf("failed to write cid field t.Piece: %w", err) + } + + return nil +} + +func (t *Query) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Piece (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Piece: %w", err) + } + + t.Piece = c + + } + return nil +} + +func (t *QueryResponse) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Status (retrieval.QueryResponseStatus) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { + return err + } + + // t.t.Size (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { + return err + } + + // t.t.MinPrice (types.BigInt) (struct) + if err := t.MinPrice.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *QueryResponse) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Status (retrieval.QueryResponseStatus) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Status = QueryResponseStatus(extra) + // t.t.Size (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + // t.t.MinPrice (types.BigInt) (struct) + + { + + if err := t.MinPrice.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + +func (t *Unixfs0Offer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Offset (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Offset))); err != nil { + return err + } + + // t.t.Size (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { + return err + } + return nil +} + +func (t *Unixfs0Offer) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Offset (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Offset = uint64(extra) + // t.t.Size (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + return nil +} + +func (t *DealProposal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Payment (api.PaymentInfo) (struct) + if err := t.Payment.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Ref (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Ref); err != nil { + return xerrors.Errorf("failed to write cid field t.Ref: %w", err) + } + + // t.t.Params (retrieval.RetParams) (struct) + if err := t.Params.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *DealProposal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Payment (api.PaymentInfo) (struct) + + { + + if err := t.Payment.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Ref (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Ref: %w", err) + } + + t.Ref = c + + } + // t.t.Params (retrieval.RetParams) (struct) + + { + + if err := t.Params.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + +func (t *DealResponse) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Status (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { + return err + } + + // t.t.Message (string) (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Message)); err != nil { + return err + } + return nil +} + +func (t *DealResponse) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Status (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Status = uint64(extra) + // t.t.Message (string) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Message = string(sval) + } + return nil +} + +func (t *Block) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Prefix ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Prefix)))); err != nil { + return err + } + if _, err := w.Write(t.Prefix); err != nil { + return err + } + + // t.t.Data ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Data)))); err != nil { + return err + } + if _, err := w.Write(t.Data); err != nil { + return err + } + return nil +} + +func (t *Block) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Prefix ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Prefix: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Prefix = make([]byte, extra) + if _, err := io.ReadFull(br, t.Prefix); err != nil { + return err + } + // t.t.Data ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Data: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Data = make([]byte, extra) + if _, err := io.ReadFull(br, t.Data); err != nil { + return err + } + return nil +} diff --git a/retrieval/client.go b/retrieval/client.go index ec086ef49..8936fbafb 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -3,22 +3,21 @@ package retrieval import ( "context" "io" - "io/ioutil" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/retrieval/discovery" @@ -45,7 +44,7 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid. } defer s.Close() - err = cborrpc.WriteCborRPC(s, Query{ + err = cborutil.WriteCborRPC(s, &Query{ Piece: data, }) if err != nil { @@ -53,15 +52,8 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid. return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } - // TODO: read deadline - rawResp, err := ioutil.ReadAll(s) - if err != nil { - log.Warn(err) - return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} - } - var resp QueryResponse - if err := cbor.DecodeInto(rawResp, &resp); err != nil { + if err := resp.UnmarshalCBOR(s); err != nil { log.Warn(err) return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } @@ -78,6 +70,7 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid. type clientStream struct { payapi payapi.PaychAPI stream network.Stream + peeker cbg.BytePeeker root cid.Cid size types.BigInt @@ -127,6 +120,7 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, cst := clientStream{ payapi: c.payapi, stream: s, + peeker: cbg.GetPeeker(s), root: root, size: types.NewInt(size), @@ -167,7 +161,7 @@ func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out return xerrors.Errorf("setting up retrieval payment: %w", err) } - deal := DealProposal{ + deal := &DealProposal{ Payment: payment, Ref: cst.root, Params: RetParams{ @@ -178,12 +172,12 @@ func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out }, } - if err := cborrpc.WriteCborRPC(cst.stream, deal); err != nil { + if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil { return err } var resp DealResponse - if err := cborrpc.ReadCborRPC(cst.stream, &resp); err != nil { + if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil { log.Error(err) return err } @@ -215,7 +209,7 @@ func (cst *clientStream) fetchBlocks(toFetch uint64, out io.Writer) error { log.Infof("block %d of %d", i+1, blocksToFetch) var block Block - if err := cborrpc.ReadCborRPC(cst.stream, &block); err != nil { + if err := cborutil.ReadCborRPC(cst.peeker, &block); err != nil { return xerrors.Errorf("reading fetchBlock response: %w", err) } diff --git a/retrieval/miner.go b/retrieval/miner.go index 350177fe2..1792880de 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -13,14 +13,19 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/storage/sectorblocks" ) +type RetrMinerApi interface { + PaychVoucherAdd(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) +} + type Miner struct { sectorBlocks *sectorblocks.SectorBlocks - full api.FullNode + full RetrMinerApi pricePerByte types.BigInt // TODO: Unseal price @@ -37,7 +42,7 @@ func NewMiner(sblks *sectorblocks.SectorBlocks, full api.FullNode) *Miner { func writeErr(stream network.Stream, err error) { log.Errorf("Retrieval deal error: %s", err) - _ = cborrpc.WriteCborRPC(stream, DealResponse{ + _ = cborutil.WriteCborRPC(stream, &DealResponse{ Status: Error, Message: err.Error(), }) @@ -47,7 +52,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { defer stream.Close() var query Query - if err := cborrpc.ReadCborRPC(stream, &query); err != nil { + if err := cborutil.ReadCborRPC(stream, &query); err != nil { writeErr(stream, err) return } @@ -58,7 +63,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { return } - answer := QueryResponse{ + answer := &QueryResponse{ Status: Unavailable, } if err == nil { @@ -69,7 +74,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { answer.Size = uint64(size) // TODO: verify on intermediate } - if err := cborrpc.WriteCborRPC(stream, answer); err != nil { + if err := cborutil.WriteCborRPC(stream, answer); err != nil { log.Errorf("Retrieval query: WriteCborRPC: %s", err) return } @@ -109,7 +114,7 @@ func (m *Miner) HandleDealStream(stream network.Stream) { func (hnd *handlerDeal) handleNext() (bool, error) { var deal DealProposal - if err := cborrpc.ReadCborRPC(hnd.stream, &deal); err != nil { + if err := cborutil.ReadCborRPC(hnd.stream, &deal); err != nil { if err == io.EOF { // client sent all deals err = nil } @@ -134,7 +139,7 @@ func (hnd *handlerDeal) handleNext() (bool, error) { // If the file isn't open (new deal stream), isn't the right file, or isn't // at the right offset, (re)open it if hnd.open != deal.Ref || hnd.at != unixfs0.Offset { - log.Infof("opening file for sending (open '%s') (@%d, want %d)", hnd.open, hnd.at, unixfs0.Offset) + log.Infof("opening file for sending (open '%s') (@%d, want %d)", deal.Ref, hnd.at, unixfs0.Offset) if err := hnd.openFile(deal); err != nil { return false, err } @@ -195,10 +200,10 @@ func (hnd *handlerDeal) openFile(deal DealProposal) error { func (hnd *handlerDeal) accept(deal DealProposal) error { unixfs0 := deal.Params.Unixfs0 - resp := DealResponse{ + resp := &DealResponse{ Status: Accepted, } - if err := cborrpc.WriteCborRPC(hnd.stream, resp); err != nil { + if err := cborutil.WriteCborRPC(hnd.stream, resp); err != nil { log.Errorf("Retrieval query: Write Accepted resp: %s", err) return err } @@ -221,12 +226,12 @@ func (hnd *handlerDeal) accept(deal DealProposal) error { return }*/ - block := Block{ + block := &Block{ Prefix: nd.Cid().Prefix().Bytes(), Data: nd.RawData(), } - if err := cborrpc.WriteCborRPC(hnd.stream, block); err != nil { + if err := cborutil.WriteCborRPC(hnd.stream, block); err != nil { return err } diff --git a/retrieval/types.go b/retrieval/types.go index a7a66ea51..28343e6b2 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -3,7 +3,6 @@ package retrieval import ( "github.com/filecoin-project/lotus/api" "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/chain/types" ) @@ -11,7 +10,7 @@ import ( const ProtocolID = "/fil/retrieval/-1.0.0" // TODO: spec const QueryProtocolID = "/fil/retrieval/qry/-1.0.0" // TODO: spec -type QueryResponseStatus int +type QueryResponseStatus uint64 const ( Available QueryResponseStatus = iota @@ -25,18 +24,6 @@ const ( Unsealing ) -func init() { - cbor.RegisterCborType(RetParams{}) - - cbor.RegisterCborType(Query{}) - cbor.RegisterCborType(QueryResponse{}) - cbor.RegisterCborType(Unixfs0Offer{}) - - cbor.RegisterCborType(DealProposal{}) - cbor.RegisterCborType(DealResponse{}) - cbor.RegisterCborType(Block{}) -} - type Query struct { Piece cid.Cid // TODO: payment @@ -69,7 +56,7 @@ type DealProposal struct { } type DealResponse struct { - Status int + Status uint64 Message string } diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go new file mode 100644 index 000000000..623cd31a9 --- /dev/null +++ b/storage/cbor_gen.go @@ -0,0 +1,574 @@ +package storage + +import ( + "fmt" + "io" + + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *SealTicket) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.BlockHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.BlockHeight))); err != nil { + return err + } + + // t.t.TicketBytes ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil { + return err + } + if _, err := w.Write(t.TicketBytes); err != nil { + return err + } + return nil +} + +func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.BlockHeight (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.BlockHeight = uint64(extra) + // t.t.TicketBytes ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.TicketBytes: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.TicketBytes = make([]byte, extra) + if _, err := io.ReadFull(br, t.TicketBytes); err != nil { + return err + } + return nil +} + +func (t *SealSeed) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.BlockHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.BlockHeight))); err != nil { + return err + } + + // t.t.TicketBytes ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil { + return err + } + if _, err := w.Write(t.TicketBytes); err != nil { + return err + } + return nil +} + +func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.BlockHeight (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.BlockHeight = uint64(extra) + // t.t.TicketBytes ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.TicketBytes: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.TicketBytes = make([]byte, extra) + if _, err := io.ReadFull(br, t.TicketBytes); err != nil { + return err + } + return nil +} + +func (t *Piece) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{132}); err != nil { + return err + } + + // t.t.DealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { + return err + } + + // t.t.Ref (string) (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Ref)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Ref)); err != nil { + return err + } + + // t.t.Size (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { + return err + } + + // t.t.CommP ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommP)))); err != nil { + return err + } + if _, err := w.Write(t.CommP); err != nil { + return err + } + return nil +} + +func (t *Piece) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 4 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.DealID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + // t.t.Ref (string) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Ref = string(sval) + } + // t.t.Size (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + // t.t.CommP ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommP: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommP = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommP); err != nil { + return err + } + return nil +} + +func (t *SectorInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{140}); err != nil { + return err + } + + // t.t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { + return err + } + + // t.t.SectorID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { + return err + } + + // t.t.Pieces ([]storage.Piece) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Pieces)))); err != nil { + return err + } + for _, v := range t.Pieces { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.t.CommC ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommC)))); err != nil { + return err + } + if _, err := w.Write(t.CommC); err != nil { + return err + } + + // t.t.CommD ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { + return err + } + if _, err := w.Write(t.CommD); err != nil { + return err + } + + // t.t.CommR ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { + return err + } + if _, err := w.Write(t.CommR); err != nil { + return err + } + + // t.t.CommRLast ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommRLast)))); err != nil { + return err + } + if _, err := w.Write(t.CommRLast); err != nil { + return err + } + + // t.t.Proof ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { + return err + } + if _, err := w.Write(t.Proof); err != nil { + return err + } + + // t.t.Ticket (storage.SealTicket) (struct) + if err := t.Ticket.MarshalCBOR(w); err != nil { + return err + } + + // t.t.PreCommitMessage (cid.Cid) (struct) + + if t.PreCommitMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.PreCommitMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.PreCommitMessage: %w", err) + } + } + + // t.t.Seed (storage.SealSeed) (struct) + if err := t.Seed.MarshalCBOR(w); err != nil { + return err + } + + // t.t.CommitMessage (cid.Cid) (struct) + + if t.CommitMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.CommitMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err) + } + } + + return nil +} + +func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 12 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.State (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.State = uint64(extra) + // t.t.SectorID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = uint64(extra) + // t.t.Pieces ([]storage.Piece) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Pieces: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Pieces = make([]Piece, extra) + } + for i := 0; i < int(extra); i++ { + + var v Piece + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Pieces[i] = v + } + + // t.t.CommC ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommC: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommC = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommC); err != nil { + return err + } + // t.t.CommD ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommD: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommD = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommD); err != nil { + return err + } + // t.t.CommR ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommR: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommR = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommR); err != nil { + return err + } + // t.t.CommRLast ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommRLast: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommRLast = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommRLast); err != nil { + return err + } + // t.t.Proof ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Proof: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Proof = make([]byte, extra) + if _, err := io.ReadFull(br, t.Proof); err != nil { + return err + } + // t.t.Ticket (storage.SealTicket) (struct) + + { + + if err := t.Ticket.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.PreCommitMessage (cid.Cid) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommitMessage: %w", err) + } + + t.PreCommitMessage = &c + } + + } + // t.t.Seed (storage.SealSeed) (struct) + + { + + if err := t.Seed.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.CommitMessage (cid.Cid) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) + } + + t.CommitMessage = &c + } + + } + return nil +} diff --git a/storage/commitment/tracker.go b/storage/commitment/tracker.go deleted file mode 100644 index 90e2459bf..000000000 --- a/storage/commitment/tracker.go +++ /dev/null @@ -1,198 +0,0 @@ -package commitment - -import ( - "context" - "fmt" - "strconv" - "strings" - "sync" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/address" - "github.com/filecoin-project/lotus/node/modules/dtypes" - dsq "github.com/ipfs/go-datastore/query" -) - -var log = logging.Logger("commitment") - -func init() { - cbor.RegisterCborType(commitment{}) -} - -var commitmentDsPrefix = datastore.NewKey("/commitments") - -type Tracker struct { - commitments datastore.Datastore - - lk sync.Mutex - - waits map[datastore.Key]chan struct{} -} - -func NewTracker(ds dtypes.MetadataDS) *Tracker { - return &Tracker{ - commitments: namespace.Wrap(ds, commitmentDsPrefix), - waits: map[datastore.Key]chan struct{}{}, - } -} - -type commitment struct { - DealIDs []uint64 - Msg cid.Cid -} - -func commitmentKey(miner address.Address, sectorId uint64) datastore.Key { - return commitmentDsPrefix.ChildString(miner.String()).ChildString(fmt.Sprintf("%d", sectorId)) -} - -func (ct *Tracker) TrackCommitSectorMsg(miner address.Address, sectorId uint64, commitMsg cid.Cid) error { - key := commitmentKey(miner, sectorId) - - ct.lk.Lock() - defer ct.lk.Unlock() - - tracking, err := ct.commitments.Get(key) - switch err { - case nil: - var comm commitment - if err := cbor.DecodeInto(tracking, &comm); err != nil { - return err - } - - if !comm.Msg.Equals(commitMsg) { - log.Errorf("commitment tracking for miner %s, sector %d: already tracking %s, got another commitment message: %s", miner, sectorId, comm.Msg, commitMsg) - } - - log.Warnf("commitment.TrackCommitSectorMsg called more than once for miner %s, sector %d, message %s", miner, sectorId, commitMsg) - - // we still want to store it - fallthrough // TODO: ideally we'd keep around both (even though we'll - // usually only need the new one) - case datastore.ErrNotFound: - comm := &commitment{Msg: commitMsg} - commB, err := cbor.DumpObject(comm) - if err != nil { - return err - } - - if err := ct.commitments.Put(key, commB); err != nil { - return err - } - - waits, ok := ct.waits[key] - if ok { - close(waits) - delete(ct.waits, key) - } - return nil - default: - return err - } -} - -func (ct *Tracker) WaitCommit(ctx context.Context, miner address.Address, sectorId uint64) (cid.Cid, error) { - key := commitmentKey(miner, sectorId) - - ct.lk.Lock() - - tracking, err := ct.commitments.Get(key) - if err != datastore.ErrNotFound { - ct.lk.Unlock() - - if err != nil { - return cid.Undef, err - } - - var comm commitment - if err := cbor.DecodeInto(tracking, &comm); err != nil { - return cid.Undef, err - } - - return comm.Msg, nil - } - - wait, ok := ct.waits[key] - if !ok { - wait = make(chan struct{}) - ct.waits[key] = wait - } - - ct.lk.Unlock() - - select { - case <-wait: - tracking, err := ct.commitments.Get(key) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to get commitment after waiting: %w", err) - } - - var comm commitment - if err := cbor.DecodeInto(tracking, &comm); err != nil { - return cid.Undef, err - } - - return comm.Msg, nil - case <-ctx.Done(): - return cid.Undef, ctx.Err() - } -} - -func (ct *Tracker) List() ([]api.SectorCommitment, error) { - out := make([]api.SectorCommitment, 0) - - ct.lk.Lock() - defer ct.lk.Unlock() - - res, err := ct.commitments.Query(dsq.Query{}) - if err != nil { - return nil, err - } - defer res.Close() - - for { - res, ok := res.NextSync() - if !ok { - break - } - - if res.Error != nil { - return nil, xerrors.Errorf("iterating commitments: %w", err) - } - - parts := strings.Split(res.Key, "/") - if len(parts) != 4 { - return nil, xerrors.Errorf("expected commitment key to be 4 parts, Key %s", res.Key) - } - - miner, err := address.NewFromString(parts[2]) - if err != nil { - return nil, xerrors.Errorf("parsing miner address: %w", err) - } - - sectorID, err := strconv.ParseInt(parts[3], 10, 64) - if err != nil { - return nil, xerrors.Errorf("parsing sector id: %w", err) - } - - var comm commitment - if err := cbor.DecodeInto(res.Value, &comm); err != nil { - return nil, xerrors.Errorf("decoding commitment %s (`% X`): %w", res.Key, res.Value, err) - } - - out = append(out, api.SectorCommitment{ - SectorID: uint64(sectorID), - Miner: miner, - CommitMsg: comm.Msg, - DealIDs: comm.DealIDs, - }) - } - - return out, nil -} diff --git a/storage/garbage.go b/storage/garbage.go new file mode 100644 index 000000000..f61ff50cc --- /dev/null +++ b/storage/garbage.go @@ -0,0 +1,140 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "math" + "math/rand" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" +) + +func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) { + if len(sizes) == 0 { + return nil, nil + } + + deals := make([]actors.StorageDeal, len(sizes)) + for i, size := range sizes { + commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) + if err != nil { + return nil, err + } + + sdp := actors.StorageDealProposal{ + PieceRef: commP[:], + PieceSize: size, + PieceSerialization: actors.SerializationUnixFSv0, + Client: m.worker, + Provider: m.maddr, + ProposalExpiration: math.MaxUint64, + Duration: math.MaxUint64 / 2, // /2 because overflows + StoragePricePerEpoch: types.NewInt(0), + StorageCollateral: types.NewInt(0), + ProposerSignature: nil, + } + + if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &sdp); err != nil { + return nil, xerrors.Errorf("signing storage deal failed: ", err) + } + + storageDeal := actors.StorageDeal{ + Proposal: sdp, + } + if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &storageDeal); err != nil { + return nil, xerrors.Errorf("signing storage deal failed: ", err) + } + + deals[i] = storageDeal + } + + params, aerr := actors.SerializeParams(&actors.PublishStorageDealsParams{ + Deals: deals, + }) + if aerr != nil { + return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", aerr) + } + + smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: m.worker, + Value: types.NewInt(0), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.PublishStorageDeals, + Params: params, + }) + if err != nil { + return nil, err + } + r, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return nil, err + } + if r.Receipt.ExitCode != 0 { + log.Error(xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode)) + } + var resp actors.PublishStorageDealResponse + if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { + return nil, err + } + if len(resp.DealIDs) != len(sizes) { + return nil, xerrors.New("got unexpected number of DealIDs from PublishStorageDeals") + } + + out := make([]Piece, len(sizes)) + + for i, size := range sizes { + name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) + ppi, err := m.sb.AddPiece(size, sectorID, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), existingPieceSizes) + if err != nil { + return nil, err + } + + existingPieceSizes = append(existingPieceSizes, size) + + out[i] = Piece{ + DealID: resp.DealIDs[i], + Ref: name, + Size: ppi.Size, + CommP: ppi.CommP[:], + } + } + + return out, nil +} + +func (m *Miner) StoreGarbageData() error { + go func() { + ctx := context.TODO() // we can't use the context from command which invokes + // this, as we run everything here async, and it's cancelled when the + // command exits + + size := sectorbuilder.UserBytesForSectorSize(m.sb.SectorSize()) + + sid, err := m.sb.AcquireSectorId() + if err != nil { + log.Errorf("%+v", err) + return + } + + pieces, err := m.storeGarbage(ctx, sid, []uint64{}, size) + if err != nil { + log.Errorf("%+v", err) + return + } + + if err := m.newSector(context.TODO(), sid, pieces[0].DealID, pieces[0].Ref, pieces[0].ppi()); err != nil { + log.Errorf("%+v", err) + return + } + }() + return nil +} diff --git a/storage/miner.go b/storage/miner.go index e563edaa4..5cc8d37ed 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -6,20 +6,18 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" "github.com/pkg/errors" - "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/storage/commitment" - "github.com/filecoin-project/lotus/storage/sector" + "github.com/filecoin-project/lotus/lib/statestore" ) var log = logging.Logger("storageminer") @@ -29,20 +27,24 @@ const PoStConfidence = 3 type Miner struct { api storageMinerApi events *events.Events + h host.Host - secst *sector.Store - commt *commitment.Tracker - - maddr address.Address - + maddr address.Address worker address.Address - h host.Host - - ds datastore.Batching - - schedLk sync.Mutex + // PoSt + postLk sync.Mutex schedPost uint64 + + // Sealing + sb *sectorbuilder.SectorBuilder + sectors *statestore.StateStore + tktFn TicketFn + + sectorIncoming chan *SectorInfo + sectorUpdated chan sectorUpdate + stop chan struct{} + stopped chan struct{} } type storageMinerApi interface { @@ -53,8 +55,8 @@ type storageMinerApi interface { StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error) StateMinerProvingPeriodEnd(context.Context, address.Address, *types.TipSet) (uint64, error) - StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error) - StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error) + StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) + StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error) StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error) @@ -66,19 +68,26 @@ type storageMinerApi interface { ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet) (*types.TipSet, error) ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) + WalletSign(context.Context, address.Address, []byte) (*types.Signature, error) WalletBalance(context.Context, address.Address) (types.BigInt, error) WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, secst *sector.Store, commt *commitment.Tracker) (*Miner, error) { +func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb *sectorbuilder.SectorBuilder, tktFn TicketFn) (*Miner, error) { return &Miner{ api: api, maddr: addr, h: h, - ds: ds, - secst: secst, - commt: commt, + sb: sb, + tktFn: tktFn, + + sectors: statestore.New(namespace.Wrap(ds, datastore.NewKey("/sectors"))), + + sectorIncoming: make(chan *SectorInfo), + sectorUpdated: make(chan sectorUpdate), + stop: make(chan struct{}), + stopped: make(chan struct{}), }, nil } @@ -89,135 +98,19 @@ func (m *Miner) Run(ctx context.Context) error { m.events = events.NewEvents(ctx, m.api) - go m.handlePostingSealedSectors(ctx) go m.beginPosting(ctx) + go m.sectorStateLoop(ctx) return nil } -func (m *Miner) commitUntrackedSectors(ctx context.Context) error { - sealed, err := m.secst.Sealed() - if err != nil { - return err +func (m *Miner) Stop(ctx context.Context) error { + close(m.stop) + select { + case <-m.stopped: + return nil + case <-ctx.Done(): + return ctx.Err() } - - chainSectors, err := m.api.StateMinerSectors(ctx, m.maddr, nil) - if err != nil { - return err - } - - onchain := map[uint64]struct{}{} - for _, chainSector := range chainSectors { - onchain[chainSector.SectorID] = struct{}{} - } - - for _, s := range sealed { - if _, ok := onchain[s.SectorID]; ok { - continue - } - - log.Warnf("Missing commitment for sector %d, committing sector", s.SectorID) - - if err := m.commitSector(ctx, s); err != nil { - log.Error("Committing uncommitted sector failed: ", err) - } - } - return nil -} - -func (m *Miner) handlePostingSealedSectors(ctx context.Context) { - incoming := m.secst.Incoming() - defer m.secst.CloseIncoming(incoming) - - if err := m.commitUntrackedSectors(ctx); err != nil { - log.Error(err) - } - - for { - select { - case sinfo, ok := <-incoming: - if !ok { - // TODO: set some state variable so that this state can be - // visible via some status command - log.Warn("sealed sector channel closed, aborting process") - return - } - - if err := m.commitSector(ctx, sinfo); err != nil { - log.Errorf("failed to commit sector: %s", err) - continue - } - - case <-ctx.Done(): - log.Warn("exiting seal posting routine") - return - } - } -} - -func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSealingStatus) error { - log.Info("committing sector") - - ssize, err := m.SectorSize(ctx) - if err != nil { - return xerrors.Errorf("failed to check out own sector size: %w", err) - } - - ok, err := sectorbuilder.VerifySeal(ssize, sinfo.CommR[:], sinfo.CommD[:], m.maddr, sinfo.Ticket.TicketBytes[:], sinfo.SectorID, sinfo.Proof) - if err != nil { - log.Error("failed to verify seal we just created: ", err) - } - if !ok { - log.Error("seal we just created failed verification") - } - - deals, err := m.secst.DealsForCommit(sinfo.SectorID) - if err != nil { - return xerrors.Errorf("getting sector deals failed: %w", err) - } - - params := &actors.OnChainSealVerifyInfo{ - CommD: sinfo.CommD[:], - CommR: sinfo.CommR[:], - Proof: sinfo.Proof, - Epoch: sinfo.Ticket.BlockHeight, - - DealIDs: deals, - SectorNumber: sinfo.SectorID, - } - enc, aerr := actors.SerializeParams(params) - if aerr != nil { - return errors.Wrap(aerr, "could not serialize commit sector parameters") - } - - msg := &types.Message{ - To: m.maddr, - From: m.worker, - Method: actors.MAMethods.CommitSector, - Params: enc, - Value: types.NewInt(0), // TODO: need to ensure sufficient collateral - GasLimit: types.NewInt(1000000 /* i dont know help */), - GasPrice: types.NewInt(1), - } - - smsg, err := m.api.MpoolPushMessage(ctx, msg) - if err != nil { - return errors.Wrap(err, "pushing message to mpool") - } - - go func() { - _, err := m.api.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - return - } - - m.beginPosting(ctx) - }() - - if err := m.commt.TrackCommitSectorMsg(m.maddr, sinfo.SectorID, smsg.Cid()); err != nil { - return xerrors.Errorf("tracking sector commitment: %w", err) - } - - return nil } func (m *Miner) runPreflightChecks(ctx context.Context) error { @@ -240,8 +133,3 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error { log.Infof("starting up miner %s, worker addr %s", m.maddr, m.worker) return nil } - -func (m *Miner) SectorSize(ctx context.Context) (uint64, error) { - // TODO: cache this - return m.api.StateMinerSectorSize(ctx, m.maddr, nil) -} diff --git a/storage/post.go b/storage/post.go index ee024acbb..3d6ac56b3 100644 --- a/storage/post.go +++ b/storage/post.go @@ -4,12 +4,15 @@ import ( "context" "time" + "github.com/ipfs/go-cid" + "go.opencensus.io/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" ) func (m *Miner) beginPosting(ctx context.Context) { @@ -30,10 +33,10 @@ func (m *Miner) beginPosting(ctx context.Context) { return } - m.schedLk.Lock() + m.postLk.Lock() if m.schedPost > 0 { log.Warnf("PoSts already running %d", m.schedPost) - m.schedLk.Unlock() + m.postLk.Unlock() return } @@ -42,10 +45,10 @@ func (m *Miner) beginPosting(ctx context.Context) { ppe, _ = actors.ProvingPeriodEnd(ppe, ts.Height()+1) m.schedPost = ppe - m.schedLk.Unlock() + m.postLk.Unlock() log.Infof("Scheduling post at height %d", ppe-build.PoStChallangeTime) - err = m.events.ChainAt(m.computePost(m.schedPost), func(ts *types.TipSet) error { // Revert + err = m.events.ChainAt(m.computePost(m.schedPost), func(ctx context.Context, ts *types.TipSet) error { // Revert // TODO: Cancel post log.Errorf("TODO: Cancel PoSt, re-run") return nil @@ -71,20 +74,20 @@ func (m *Miner) scheduleNextPost(ppe uint64) { ppe = headPPE } - m.schedLk.Lock() + m.postLk.Lock() if m.schedPost >= ppe { // this probably can't happen log.Errorw("PoSt already scheduled", "schedPost", m.schedPost, "ppe", ppe) - m.schedLk.Unlock() + m.postLk.Unlock() return } m.schedPost = ppe - m.schedLk.Unlock() + m.postLk.Unlock() log.Infow("scheduling PoSt", "post-height", ppe-build.PoStChallangeTime, "height", ts.Height(), "ppe", ppe, "proving-period", provingPeriod) - err = m.events.ChainAt(m.computePost(ppe), func(ts *types.TipSet) error { // Revert + err = m.events.ChainAt(m.computePost(ppe), func(ctx context.Context, ts *types.TipSet) error { // Revert // TODO: Cancel post log.Errorf("TODO: Cancel PoSt, re-run") return nil @@ -96,9 +99,164 @@ func (m *Miner) scheduleNextPost(ppe uint64) { } } -func (m *Miner) computePost(ppe uint64) func(ts *types.TipSet, curH uint64) error { +type post struct { + m *Miner + + ppe uint64 + ts *types.TipSet + + // prep + sset []*api.ChainSectorInfo + r []byte + + // run + proof []byte + + // commit + smsg cid.Cid +} + +func (p *post) doPost(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.computePost") + defer span.End() + + if err := p.preparePost(ctx); err != nil { + return err + } + + if err := p.runPost(ctx); err != nil { + return err + } + + if err := p.commitPost(ctx); err != nil { + return err + } + + if err := p.waitCommit(ctx); err != nil { + return err + } + + return nil +} + +func (p *post) preparePost(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.preparePost") + defer span.End() + + sset, err := p.m.api.StateMinerProvingSet(ctx, p.m.maddr, p.ts) + if err != nil { + return xerrors.Errorf("failed to get proving set for miner: %w", err) + } + p.sset = sset + + // Compute how many blocks back we have to look from the given tipset for the PoSt challenge + challengeLookback := int((int64(p.ts.Height()) - int64(p.ppe)) + int64(build.PoStChallangeTime) + int64(build.PoStRandomnessLookback)) + r, err := p.m.api.ChainGetRandomness(ctx, p.ts, nil, challengeLookback) + if err != nil { + return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", p.ts.Height(), p.ppe, err) + } + p.r = r + + return nil +} + +func (p *post) sortedSectorInfo() sectorbuilder.SortedSectorInfo { + sbsi := make([]sectorbuilder.SectorInfo, len(p.sset)) + for k, sector := range p.sset { + var commR [sectorbuilder.CommLen]byte + copy(commR[:], sector.CommR) + + sbsi[k] = sectorbuilder.SectorInfo{ + SectorID: sector.SectorID, + CommR: commR, + } + } + + return sectorbuilder.NewSortedSectorInfo(sbsi) +} + +func (p *post) runPost(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.runPost") + defer span.End() + + log.Infow("running PoSt", "delayed-by", + int64(p.ts.Height())-(int64(p.ppe)-int64(build.PoStChallangeTime)), + "chain-random", p.r, "ppe", p.ppe, "height", p.ts.Height()) + + tsStart := time.Now() + var faults []uint64 // TODO + + var seed [32]byte + copy(seed[:], p.r) + + proof, err := p.m.sb.GeneratePoSt(p.sortedSectorInfo(), seed, faults) + if err != nil { + return xerrors.Errorf("running post failed: %w", err) + } + elapsed := time.Since(tsStart) + + p.proof = proof + log.Infow("submitting PoSt", "pLen", len(proof), "elapsed", elapsed) + + return nil +} + +func (p *post) commitPost(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.commitPost") + defer span.End() + + params := &actors.SubmitPoStParams{ + Proof: p.proof, + DoneSet: types.BitFieldFromSet(nil), + } + + enc, aerr := actors.SerializeParams(params) + if aerr != nil { + return xerrors.Errorf("could not serialize submit post parameters: %w", aerr) + } + + msg := &types.Message{ + To: p.m.maddr, + From: p.m.worker, + Method: actors.MAMethods.SubmitPoSt, + Params: enc, + Value: types.NewInt(1000), // currently hard-coded late fee in actor, returned if not late + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + log.Info("mpush") + + smsg, err := p.m.api.MpoolPushMessage(ctx, msg) + if err != nil { + return xerrors.Errorf("pushing message to mpool: %w", err) + } + p.smsg = smsg.Cid() + + return nil +} + +func (p *post) waitCommit(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.waitPost") + defer span.End() + + log.Infof("Waiting for post %s to appear on chain", p.smsg) + + // make sure it succeeds... + rec, err := p.m.api.StateWaitMsg(ctx, p.smsg) + if err != nil { + return err + } + if rec.Receipt.ExitCode != 0 { + log.Warnf("SubmitPoSt EXIT: %d", rec.Receipt.ExitCode) + // TODO: Do something + } + return nil +} + +func (m *Miner) computePost(ppe uint64) func(ctx context.Context, ts *types.TipSet, curH uint64) error { called := 0 - return func(ts *types.TipSet, curH uint64) error { + return func(ctx context.Context, ts *types.TipSet, curH uint64) error { called++ if called > 1 { log.Errorw("BUG: computePost callback called again", "ppe", ppe, @@ -106,77 +264,21 @@ func (m *Miner) computePost(ppe uint64) func(ts *types.TipSet, curH uint64) erro return nil } - ctx := context.TODO() - - sset, err := m.api.StateMinerProvingSet(ctx, m.maddr, ts) - if err != nil { - return xerrors.Errorf("failed to get proving set for miner: %w", err) - } - - r, err := m.api.ChainGetRandomness(ctx, ts, nil, int(int64(ts.Height())-int64(ppe)+int64(build.PoStChallangeTime)+int64(build.PoStRandomnessLookback))) // TODO: review: check math - if err != nil { - return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", ts.Height(), ppe, err) - } - - log.Infow("running PoSt", "delayed-by", - int64(ts.Height())-(int64(ppe)-int64(build.PoStChallangeTime)), - "chain-random", r, "ppe", ppe, "height", ts.Height()) - - tsStart := time.Now() - var faults []uint64 - proof, err := m.secst.RunPoSt(ctx, sset, r, faults) - if err != nil { - return xerrors.Errorf("running post failed: %w", err) - } - elapsed := time.Since(tsStart) - - log.Infow("submitting PoSt", "pLen", len(proof), "elapsed", elapsed) - - params := &actors.SubmitPoStParams{ - Proof: proof, - DoneSet: types.BitFieldFromSet(nil), - } - - enc, aerr := actors.SerializeParams(params) - if aerr != nil { - return xerrors.Errorf("could not serialize submit post parameters: %w", err) - } - - msg := &types.Message{ - To: m.maddr, - From: m.worker, - Method: actors.MAMethods.SubmitPoSt, - Params: enc, - Value: types.NewInt(1000), // currently hard-coded late fee in actor, returned if not late - GasLimit: types.NewInt(1000000 /* i dont know help */), - GasPrice: types.NewInt(1), - } - - log.Info("mpush") - - smsg, err := m.api.MpoolPushMessage(ctx, msg) - if err != nil { - return xerrors.Errorf("pushing message to mpool: %w", err) - } - - log.Infof("Waiting for post %s to appear on chain", smsg.Cid()) - - // make sure it succeeds... - rec, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + err := (&post{ + m: m, + ppe: ppe, + ts: ts, + }).doPost(ctx) if err != nil { return err } - if rec.Receipt.ExitCode != 0 { - log.Warnf("SubmitPoSt EXIT: %d", rec.Receipt.ExitCode) - // TODO: Do something - } m.scheduleNextPost(ppe + build.ProvingPeriodDuration) return nil } } -func sectorIdList(si []*api.SectorInfo) []uint64 { +func sectorIdList(si []*api.ChainSectorInfo) []uint64 { out := make([]uint64, len(si)) for i, s := range si { out[i] = s.SectorID diff --git a/storage/sealing.go b/storage/sealing.go new file mode 100644 index 000000000..77ce1312c --- /dev/null +++ b/storage/sealing.go @@ -0,0 +1,254 @@ +package storage + +import ( + "context" + "io" + + cid "github.com/ipfs/go-cid" + xerrors "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/lib/sectorbuilder" +) + +type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) + +type SealTicket struct { + BlockHeight uint64 + TicketBytes []byte +} + +func (t *SealTicket) SB() sectorbuilder.SealTicket { + out := sectorbuilder.SealTicket{BlockHeight: t.BlockHeight} + copy(out.TicketBytes[:], t.TicketBytes) + return out +} + +type SealSeed struct { + BlockHeight uint64 + TicketBytes []byte +} + +func (t *SealSeed) SB() sectorbuilder.SealSeed { + out := sectorbuilder.SealSeed{BlockHeight: t.BlockHeight} + copy(out.TicketBytes[:], t.TicketBytes) + return out +} + +type Piece struct { + DealID uint64 + Ref string + + Size uint64 + CommP []byte +} + +func (p *Piece) ppi() (out sectorbuilder.PublicPieceInfo) { + out.Size = p.Size + copy(out.CommP[:], p.CommP) + return out +} + +type SectorInfo struct { + State api.SectorState + SectorID uint64 + + // Packing + + Pieces []Piece + + // PreCommit + CommC []byte + CommD []byte + CommR []byte + CommRLast []byte + Proof []byte + Ticket SealTicket + + PreCommitMessage *cid.Cid + + // PreCommitted + Seed SealSeed + + // Committing + CommitMessage *cid.Cid +} + +type sectorUpdate struct { + newState api.SectorState + id uint64 + err error + mut func(*SectorInfo) +} + +func (t *SectorInfo) pieceInfos() []sectorbuilder.PublicPieceInfo { + out := make([]sectorbuilder.PublicPieceInfo, len(t.Pieces)) + for i, piece := range t.Pieces { + out[i] = piece.ppi() + } + return out +} + +func (t *SectorInfo) deals() []uint64 { + out := make([]uint64, len(t.Pieces)) + for i, piece := range t.Pieces { + out[i] = piece.DealID + } + return out +} + +func (t *SectorInfo) refs() []string { + out := make([]string, len(t.Pieces)) + for i, piece := range t.Pieces { + out[i] = piece.Ref + } + return out +} + +func (t *SectorInfo) existingPieces() []uint64 { + out := make([]uint64, len(t.Pieces)) + for i, piece := range t.Pieces { + out[i] = piece.Size + } + return out +} + +func (t *SectorInfo) rspco() sectorbuilder.RawSealPreCommitOutput { + var out sectorbuilder.RawSealPreCommitOutput + + copy(out.CommC[:], t.CommC) + copy(out.CommD[:], t.CommD) + copy(out.CommR[:], t.CommR) + copy(out.CommRLast[:], t.CommRLast) + + return out +} + +func (m *Miner) sectorStateLoop(ctx context.Context) { + // TODO: restore state + + go func() { + defer log.Warn("quitting deal provider loop") + defer close(m.stopped) + + for { + select { + case sector := <-m.sectorIncoming: + m.onSectorIncoming(sector) + case update := <-m.sectorUpdated: + m.onSectorUpdated(ctx, update) + case <-m.stop: + return + } + } + }() +} + +func (m *Miner) onSectorIncoming(sector *SectorInfo) { + has, err := m.sectors.Has(sector.SectorID) + if err != nil { + return + } + if has { + log.Warnf("SealPiece called more than once for sector %d", sector.SectorID) + return + } + + if err := m.sectors.Begin(sector.SectorID, sector); err != nil { + // We may have re-sent the proposal + log.Errorf("deal tracking failed: %s", err) + m.failSector(sector.SectorID, err) + return + } + + go func() { + select { + case m.sectorUpdated <- sectorUpdate{ + newState: api.Packing, + id: sector.SectorID, + }: + case <-m.stop: + log.Warn("failed to send incoming sector update, miner shutting down") + } + }() +} + +func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { + log.Infof("Sector %d updated state to %s", update.id, api.SectorStateStr(update.newState)) + var sector SectorInfo + err := m.sectors.Mutate(update.id, func(s *SectorInfo) error { + s.State = update.newState + if update.mut != nil { + update.mut(s) + } + sector = *s + return nil + }) + if update.err != nil { + log.Errorf("sector %d failed: %s", update.id, update.err) + m.failSector(update.id, update.err) + return + } + if err != nil { + m.failSector(update.id, err) + return + } + + switch update.newState { + case api.Packing: + m.handle(ctx, sector, m.finishPacking, api.Unsealed) + case api.Unsealed: + m.handle(ctx, sector, m.sealPreCommit, api.PreCommitting) + case api.PreCommitting: + m.handle(ctx, sector, m.preCommit, api.PreCommitted) + case api.PreCommitted: + m.handle(ctx, sector, m.preCommitted, api.SectorNoUpdate) + case api.Committing: + m.handle(ctx, sector, m.committing, api.Proving) + case api.SectorNoUpdate: // noop + default: + log.Errorf("unexpected sector update state: %d", update.newState) + } +} + +func (m *Miner) failSector(id uint64, err error) { + log.Errorf("sector %d error: %+v", id, err) +} + +func (m *Miner) SealPiece(ctx context.Context, ref string, size uint64, r io.Reader, dealID uint64) (uint64, error) { + log.Infof("Seal piece for deal %d", dealID) + + sid, err := m.sb.AcquireSectorId() // TODO: Put more than one thing in a sector + if err != nil { + return 0, xerrors.Errorf("acquiring sector ID: %w", err) + } + + ppi, err := m.sb.AddPiece(size, sid, r, []uint64{}) + if err != nil { + return 0, xerrors.Errorf("adding piece to sector: %w", err) + } + + return sid, m.newSector(ctx, sid, dealID, ref, ppi) +} + +func (m *Miner) newSector(ctx context.Context, sid uint64, dealID uint64, ref string, ppi sectorbuilder.PublicPieceInfo) error { + si := &SectorInfo{ + SectorID: sid, + + Pieces: []Piece{ + { + DealID: dealID, + Ref: ref, + + Size: ppi.Size, + CommP: ppi.CommP[:], + }, + }, + } + select { + case m.sectorIncoming <- si: + return nil + case <-ctx.Done(): + return xerrors.Errorf("failed to submit sector for sealing, queue full: %w", ctx.Err()) + } +} diff --git a/storage/sealing_utils.go b/storage/sealing_utils.go new file mode 100644 index 000000000..d8d5225bb --- /dev/null +++ b/storage/sealing_utils.go @@ -0,0 +1,57 @@ +package storage + +import ( + "math/bits" + + "github.com/filecoin-project/lotus/lib/sectorbuilder" +) + +func fillersFromRem(toFill uint64) ([]uint64, error) { + // Convert to in-sector bytes for easier math: + // + // Sector size to user bytes ratio is constant, e.g. for 1024B we have 1016B + // of user-usable data. + // + // (1024/1016 = 128/127) + // + // Given that we can get sector size by simply adding 1/127 of the user + // bytes + // + // (we convert to sector bytes as they are nice round binary numbers) + + toFill += toFill / 127 + + // We need to fill the sector with pieces that are powers of 2. Conveniently + // computers store numbers in binary, which means we can look at 1s to get + // all the piece sizes we need to fill the sector. It also means that number + // of pieces is the number of 1s in the number of remaining bytes to fill + out := make([]uint64, bits.OnesCount64(toFill)) + for i := range out { + // Extract the next lowest non-zero bit + next := bits.TrailingZeros64(toFill) + psize := uint64(1) << next + // e.g: if the number is 0b010100, psize will be 0b000100 + + // set that bit to 0 by XORing it, so the next iteration looks at the + // next bit + toFill ^= psize + + // Add the piece size to the list of pieces we need to create + out[i] = sectorbuilder.UserBytesForSectorSize(psize) + } + return out, nil +} + +func (m *Miner) ListSectors() ([]SectorInfo, error) { + var sectors []SectorInfo + if err := m.sectors.List(§ors); err != nil { + return nil, err + } + return sectors, nil +} + +func (m *Miner) GetSectorInfo(sid uint64) (SectorInfo, error) { + var out SectorInfo + err := m.sectors.Get(sid, &out) + return out, err +} diff --git a/storage/sealing_utils_test.go b/storage/sealing_utils_test.go new file mode 100644 index 000000000..6a975e664 --- /dev/null +++ b/storage/sealing_utils_test.go @@ -0,0 +1,46 @@ +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/filecoin-project/lotus/lib/sectorbuilder" +) + +func testFill(t *testing.T, n uint64, exp []uint64) { + f, err := fillersFromRem(n) + assert.NoError(t, err) + assert.Equal(t, exp, f) + + var sum uint64 + for _, u := range f { + sum += u + } + assert.Equal(t, n, sum) +} + +func TestFillersFromRem(t *testing.T) { + for i := 8; i < 32; i++ { + // single + ub := sectorbuilder.UserBytesForSectorSize(uint64(1) << i) + testFill(t, ub, []uint64{ub}) + + // 2 + ub = sectorbuilder.UserBytesForSectorSize(uint64(5) << i) + ub1 := sectorbuilder.UserBytesForSectorSize(uint64(1) << i) + ub3 := sectorbuilder.UserBytesForSectorSize(uint64(4) << i) + testFill(t, ub, []uint64{ub1, ub3}) + + // 4 + ub = sectorbuilder.UserBytesForSectorSize(uint64(15) << i) + ub2 := sectorbuilder.UserBytesForSectorSize(uint64(2) << i) + ub4 := sectorbuilder.UserBytesForSectorSize(uint64(8) << i) + testFill(t, ub, []uint64{ub1, ub2, ub3, ub4}) + + // different 2 + ub = sectorbuilder.UserBytesForSectorSize(uint64(9) << i) + testFill(t, ub, []uint64{ub1, ub4}) + } + +} diff --git a/storage/sector/store.go b/storage/sector/store.go deleted file mode 100644 index 30363ee67..000000000 --- a/storage/sector/store.go +++ /dev/null @@ -1,344 +0,0 @@ -package sector - -import ( - "context" - "fmt" - "io" - "sync" - "time" - - "github.com/filecoin-project/go-sectorbuilder/sealing_state" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -func init() { - cbor.RegisterCborType(dealMapping{}) -} - -var log = logging.Logger("sectorstore") - -var sectorDealsPrefix = datastore.NewKey("/sectordeals") - -type dealMapping struct { - DealIDs []uint64 - Committed bool -} - -type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) - -// TODO: eventually handle sector storage here instead of in rust-sectorbuilder -type Store struct { - waitingLk sync.Mutex - - sb *sectorbuilder.SectorBuilder - tktFn TicketFn - - dealsLk sync.Mutex - deals datastore.Datastore - - waiting map[uint64]chan struct{} - incoming []chan sectorbuilder.SectorSealingStatus - // TODO: outdated chan - - closeCh chan struct{} -} - -func NewStore(sb *sectorbuilder.SectorBuilder, ds dtypes.MetadataDS, tktFn TicketFn) *Store { - return &Store{ - sb: sb, - tktFn: tktFn, - deals: namespace.Wrap(ds, sectorDealsPrefix), - waiting: map[uint64]chan struct{}{}, - closeCh: make(chan struct{}), - } -} - -func (s *Store) Service() { - go s.service() -} - -func (s *Store) poll() { - log.Debug("polling for sealed sectors...") - - // get a list of sectors to poll - s.waitingLk.Lock() - toPoll := make([]uint64, 0, len(s.waiting)) - - for id := range s.waiting { - toPoll = append(toPoll, id) - } - s.waitingLk.Unlock() - - var done []sectorbuilder.SectorSealingStatus - - // check status of each - for _, sec := range toPoll { - status, err := s.sb.SealStatus(sec) - if err != nil { - log.Errorf("getting seal status: %s", err) - continue - } - - if status.State == sealing_state.Sealed { - done = append(done, status) - } - } - - // send updates - s.waitingLk.Lock() - for _, sector := range done { - watch, ok := s.waiting[sector.SectorID] - if ok { - close(watch) - delete(s.waiting, sector.SectorID) - } - for _, c := range s.incoming { - c <- sector // TODO: ctx! - } - } - s.waitingLk.Unlock() -} - -func (s *Store) restartSealing() { - sectors, err := s.sb.GetAllStagedSectors() - if err != nil { - return - } - - for _, sid := range sectors { - status, err := s.sb.SealStatus(sid) - if err != nil { - return - } - - if status.State != sealing_state.Paused { - continue - } - - log.Infof("Sector %d is in paused state, resuming sealing", sid) - go func() { - // TODO: when we refactor wait-for-seal below, care about this output too - // (see SealSector below) - _, err := s.sb.ResumeSealSector(sid) - if err != nil { - return - } - }() - } -} - -func (s *Store) service() { - poll := time.Tick(5 * time.Second) - - s.restartSealing() - - for { - select { - case <-poll: - s.poll() - case <-s.closeCh: - s.waitingLk.Lock() - for _, c := range s.incoming { - close(c) - } - s.waitingLk.Unlock() - return - } - } -} - -func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { - sectorID, err = s.sb.AddPiece(ref, size, r) - if err != nil { - return 0, err - } - - s.waitingLk.Lock() - _, exists := s.waiting[sectorID] - if !exists { // pieces can share sectors - s.waiting[sectorID] = make(chan struct{}) - } - s.waitingLk.Unlock() - - s.dealsLk.Lock() - defer s.dealsLk.Unlock() - - k := datastore.NewKey(fmt.Sprint(sectorID)) - e, err := s.deals.Get(k) - var deals dealMapping - switch err { - case nil: - if err := cbor.DecodeInto(e, &deals); err != nil { - return 0, err - } - if deals.Committed { - return 0, xerrors.Errorf("sector %d already committed", sectorID) - } - fallthrough - case datastore.ErrNotFound: - deals.DealIDs = append(deals.DealIDs, dealIDs...) - d, err := cbor.DumpObject(&deals) - if err != nil { - return 0, err - } - if err := s.deals.Put(k, d); err != nil { - return 0, err - } - default: - return 0, err - } - - return sectorID, nil -} - -func (s *Store) DealsForCommit(sectorID uint64) ([]uint64, error) { - s.dealsLk.Lock() - defer s.dealsLk.Unlock() - - k := datastore.NewKey(fmt.Sprint(sectorID)) - e, err := s.deals.Get(k) - - switch err { - case nil: - var deals dealMapping - if err := cbor.DecodeInto(e, &deals); err != nil { - return nil, err - } - if deals.Committed { - log.Errorf("getting deal IDs for sector %d: sector already marked as committed", sectorID) - } - - deals.Committed = true - d, err := cbor.DumpObject(&deals) - if err != nil { - return nil, err - } - if err := s.deals.Put(k, d); err != nil { - return nil, err - } - - return deals.DealIDs, nil - case datastore.ErrNotFound: - log.Errorf("getting deal IDs for sector %d failed: %s", err) - return []uint64{}, nil - default: - return nil, err - } -} - -func (s *Store) SealSector(ctx context.Context, sectorID uint64) error { - tkt, err := s.tktFn(ctx) - if err != nil { - return err - } - - // TODO: That's not async, is it? - // - If not then we probably can drop this wait-for-seal hack below - _, err = s.sb.SealSector(sectorID, *tkt) - if err != nil { - return err - } - return nil -} - -func (s *Store) CloseIncoming(c <-chan sectorbuilder.SectorSealingStatus) { - s.waitingLk.Lock() - var at = -1 - for i, ch := range s.incoming { - if ch == c { - at = i - } - } - if at == -1 { - s.waitingLk.Unlock() - return - } - if len(s.incoming) > 1 { - last := len(s.incoming) - 1 - s.incoming[at] = s.incoming[last] - s.incoming[last] = nil - } - s.incoming = s.incoming[:len(s.incoming)-1] - s.waitingLk.Unlock() -} - -func (s *Store) Incoming() <-chan sectorbuilder.SectorSealingStatus { - ch := make(chan sectorbuilder.SectorSealingStatus, 8) - s.waitingLk.Lock() - s.incoming = append(s.incoming, ch) - s.waitingLk.Unlock() - return ch -} - -func (s *Store) WaitSeal(ctx context.Context, sector uint64) (sectorbuilder.SectorSealingStatus, error) { - s.waitingLk.Lock() - watch, ok := s.waiting[sector] - s.waitingLk.Unlock() - if ok { - select { - case <-watch: - case <-ctx.Done(): - return sectorbuilder.SectorSealingStatus{}, ctx.Err() - } - } - - return s.sb.SealStatus(sector) -} - -func (s *Store) Sealed() ([]sectorbuilder.SectorSealingStatus, error) { - l, err := s.sb.GetAllStagedSectors() - if err != nil { - return nil, err - } - - out := make([]sectorbuilder.SectorSealingStatus, 0) - for _, sid := range l { - status, err := s.sb.SealStatus(sid) - if err != nil { - return nil, err - } - - if status.State != sealing_state.Sealed { - continue - } - out = append(out, status) - } - - return out, nil -} - -func (s *Store) RunPoSt(ctx context.Context, sectors []*api.SectorInfo, r []byte, faults []uint64) ([]byte, error) { - sbsi := make([]sectorbuilder.SectorInfo, len(sectors)) - for k, sector := range sectors { - var commR [sectorbuilder.CommLen]byte - if copy(commR[:], sector.CommR) != sectorbuilder.CommLen { - return nil, xerrors.Errorf("commR too short, %d bytes", len(sector.CommR)) - } - - sbsi[k] = sectorbuilder.SectorInfo{ - SectorID: sector.SectorID, - CommR: commR, - } - } - - ssi := sectorbuilder.NewSortedSectorInfo(sbsi) - - var seed [sectorbuilder.CommLen]byte - if copy(seed[:], r) != sectorbuilder.CommLen { - return nil, xerrors.Errorf("random seed too short, %d bytes", len(r)) - } - - return s.sb.GeneratePoSt(ssi, seed, faults) -} - -func (s *Store) Stop() { - close(s.closeCh) -} diff --git a/storage/sector_states.go b/storage/sector_states.go new file mode 100644 index 000000000..d91e3cead --- /dev/null +++ b/storage/sector_states.go @@ -0,0 +1,231 @@ +package storage + +import ( + "context" + + "github.com/pkg/errors" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" +) + +type providerHandlerFunc func(ctx context.Context, deal SectorInfo) (func(*SectorInfo), error) + +func (m *Miner) handle(ctx context.Context, sector SectorInfo, cb providerHandlerFunc, next api.SectorState) { + go func() { + mut, err := cb(ctx, sector) + + if err == nil && next == api.SectorNoUpdate { + return + } + + select { + case m.sectorUpdated <- sectorUpdate{ + newState: next, + id: sector.SectorID, + err: err, + mut: mut, + }: + case <-m.stop: + } + }() +} + +func (m *Miner) finishPacking(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + log.Infow("performing filling up rest of the sector...", "sector", sector.SectorID) + + var allocated uint64 + for _, piece := range sector.Pieces { + allocated += piece.Size + } + + ubytes := sectorbuilder.UserBytesForSectorSize(m.sb.SectorSize()) + + if allocated > ubytes { + return nil, xerrors.Errorf("too much data in sector: %d > %d", allocated, ubytes) + } + + fillerSizes, err := fillersFromRem(ubytes - allocated) + if err != nil { + return nil, err + } + + if len(fillerSizes) > 0 { + log.Warnf("Creating %d filler pieces for sector %d", len(fillerSizes), sector.SectorID) + } + + pieces, err := m.storeGarbage(ctx, sector.SectorID, sector.existingPieces(), fillerSizes...) + if err != nil { + return nil, xerrors.Errorf("filling up the sector (%v): %w", fillerSizes, err) + } + + return func(info *SectorInfo) { + info.Pieces = append(info.Pieces, pieces...) + }, nil +} + +func (m *Miner) sealPreCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + log.Infow("performing sector replication...", "sector", sector.SectorID) + ticket, err := m.tktFn(ctx) + if err != nil { + return nil, err + } + + rspco, err := m.sb.SealPreCommit(sector.SectorID, *ticket, sector.pieceInfos()) + if err != nil { + return nil, xerrors.Errorf("seal pre commit failed: %w", err) + } + + return func(info *SectorInfo) { + info.CommC = rspco.CommC[:] + info.CommD = rspco.CommD[:] + info.CommR = rspco.CommR[:] + info.CommRLast = rspco.CommRLast[:] + info.Ticket = SealTicket{ + BlockHeight: ticket.BlockHeight, + TicketBytes: ticket.TicketBytes[:], + } + }, nil +} + +func (m *Miner) preCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + params := &actors.SectorPreCommitInfo{ + SectorNumber: sector.SectorID, + + CommR: sector.CommR, + SealEpoch: sector.Ticket.BlockHeight, + DealIDs: sector.deals(), + } + enc, aerr := actors.SerializeParams(params) + if aerr != nil { + return nil, xerrors.Errorf("could not serialize commit sector parameters: %w", aerr) + } + + msg := &types.Message{ + To: m.maddr, + From: m.worker, + Method: actors.MAMethods.PreCommitSector, + Params: enc, + Value: types.NewInt(0), // TODO: need to ensure sufficient collateral + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + log.Info("submitting precommit for sector: ", sector.SectorID) + smsg, err := m.api.MpoolPushMessage(ctx, msg) + if err != nil { + return nil, xerrors.Errorf("pushing message to mpool: %w", err) + } + + return func(info *SectorInfo) { + mcid := smsg.Cid() + info.PreCommitMessage = &mcid + }, nil +} + +func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + // would be ideal to just use the events.Called handler, but it wouldnt be able to handle individual message timeouts + log.Info("Sector precommitted: ", sector.SectorID) + mw, err := m.api.StateWaitMsg(ctx, *sector.PreCommitMessage) + if err != nil { + return nil, err + } + + if mw.Receipt.ExitCode != 0 { + log.Error("sector precommit failed: ", mw.Receipt.ExitCode) + return nil, err + } + log.Info("precommit message landed on chain: ", sector.SectorID) + + randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay - 1 // -1 because of how the messages are applied + log.Infof("precommit for sector %d made it on chain, will start proof computation at height %d", sector.SectorID, randHeight) + + err = m.events.ChainAt(func(ctx context.Context, ts *types.TipSet, curH uint64) error { + rand, err := m.api.ChainGetRandomness(ctx, ts, nil, int(ts.Height()-randHeight)) + if err != nil { + return xerrors.Errorf("failed to get randomness for computing seal proof: %w", err) + } + + m.sectorUpdated <- sectorUpdate{ + newState: api.Committing, + id: sector.SectorID, + mut: func(info *SectorInfo) { + info.Seed = SealSeed{ + BlockHeight: randHeight, + TicketBytes: rand, + } + }, + } + + return nil + }, func(ctx context.Context, ts *types.TipSet) error { + log.Warn("revert in interactive commit sector step") + return nil + }, 3, mw.TipSet.Height()+build.InteractivePoRepDelay) + if err != nil { + log.Warn("waitForPreCommitMessage ChainAt errored: ", err) + } + + return nil, nil +} + +func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + log.Info("scheduling seal proof computation...") + + proof, err := m.sb.SealCommit(sector.SectorID, sector.Ticket.SB(), sector.Seed.SB(), sector.pieceInfos(), sector.refs(), sector.rspco()) + if err != nil { + return nil, xerrors.Errorf("computing seal proof failed: %w", err) + } + + // TODO: Consider splitting states and persist proof for faster recovery + + params := &actors.SectorProveCommitInfo{ + Proof: proof, + SectorID: sector.SectorID, + DealIDs: sector.deals(), + } + + enc, aerr := actors.SerializeParams(params) + if aerr != nil { + return nil, xerrors.Errorf("could not serialize commit sector parameters: %w", aerr) + } + + msg := &types.Message{ + To: m.maddr, + From: m.worker, + Method: actors.MAMethods.ProveCommitSector, + Params: enc, + Value: types.NewInt(0), // TODO: need to ensure sufficient collateral + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + smsg, err := m.api.MpoolPushMessage(ctx, msg) + if err != nil { + log.Error(errors.Wrap(err, "pushing message to mpool")) + } + + // TODO: Separate state before this wait, so we persist message cid? + + mw, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return nil, xerrors.Errorf("failed to wait for porep inclusion: %w", err) + } + + if mw.Receipt.ExitCode != 0 { + log.Errorf("UNHANDLED: submitting sector proof failed (t:%x; s:%x(%d); p:%x)", sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, params.Proof) + return nil, xerrors.New("UNHANDLED: submitting sector proof failed") + } + + m.beginPosting(ctx) + + return func(info *SectorInfo) { + mcid := smsg.Cid() + info.CommitMessage = &mcid + info.Proof = proof + }, nil +} diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 28b864e7a..e1f64ae3e 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -1,25 +1,29 @@ package sectorblocks import ( + "bytes" "context" "errors" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/ipfs/go-datastore/namespace" - "github.com/ipfs/go-datastore/query" - blockstore "github.com/ipfs/go-ipfs-blockstore" - ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-unixfs" + "io" "sync" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" + blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" files "github.com/ipfs/go-ipfs-files" - cbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-unixfs" + "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/storage/sector" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/lib/cborutil" + "github.com/filecoin-project/lotus/lib/padreader" + "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/storage" ) type SealSerialization uint8 @@ -34,7 +38,7 @@ var imBlocksPrefix = datastore.NewKey("/intermediate") var ErrNotFound = errors.New("not found") type SectorBlocks struct { - *sector.Store + *storage.Miner intermediate blockstore.Blockstore // holds intermediate nodes TODO: consider combining with the staging blockstore @@ -43,9 +47,9 @@ type SectorBlocks struct { keyLk sync.Mutex } -func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { +func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { sbc := &SectorBlocks{ - Store: sectst, + Miner: miner, intermediate: blockstore.NewBlockstore(namespace.Wrap(ds, imBlocksPrefix)), @@ -73,14 +77,14 @@ type UnixfsReader interface { type refStorer struct { blockReader UnixfsReader - writeRef func(cid cid.Cid, pieceRef string, offset uint64, size uint32) error + writeRef func(cid cid.Cid, pieceRef string, offset uint64, size uint64) error intermediate blockstore.Blockstore pieceRef string remaining []byte } -func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, size uint32) error { +func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, size uint64) error { st.keyLk.Lock() // TODO: make this multithreaded defer st.keyLk.Unlock() @@ -89,25 +93,25 @@ func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, si err = nil } if err != nil { - return err + return xerrors.Errorf("getting existing refs: %w", err) } - var refs []api.SealedRef + var refs api.SealedRefs if len(v) > 0 { - if err := cbor.DecodeInto(v, &refs); err != nil { - return err + if err := cborutil.ReadCborRPC(bytes.NewReader(v), &refs); err != nil { + return xerrors.Errorf("decoding existing refs: %w", err) } } - refs = append(refs, api.SealedRef{ + refs.Refs = append(refs.Refs, api.SealedRef{ Piece: pieceRef, Offset: offset, Size: size, }) - newRef, err := cbor.DumpObject(&refs) + newRef, err := cborutil.Dump(&refs) if err != nil { - return err + return xerrors.Errorf("serializing refs: %w", err) } return st.keys.Put(dshelp.CidToDsKey(cid), newRef) // TODO: batch somehow } @@ -128,20 +132,23 @@ func (r *refStorer) Read(p []byte) (n int, err error) { for { data, offset, nd, err := r.blockReader.ReadBlock(context.TODO()) if err != nil { - return 0, err + if err == io.EOF { + return 0, io.EOF + } + return 0, xerrors.Errorf("reading block: %w", err) } if len(data) == 0 { // TODO: batch // TODO: GC if err := r.intermediate.Put(nd); err != nil { - return 0, err + return 0, xerrors.Errorf("storing intermediate node: %w", err) } continue } - if err := r.writeRef(nd.Cid(), r.pieceRef, offset, uint32(len(data))); err != nil { - return 0, err + if err := r.writeRef(nd.Cid(), r.pieceRef, offset, uint64(len(data))); err != nil { + return 0, xerrors.Errorf("writing ref: %w", err) } read := copy(p, data) @@ -153,7 +160,7 @@ func (r *refStorer) Read(p []byte) (n int, err error) { } } -func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint64) (sectorID uint64, err error) { +func (st *SectorBlocks) AddUnixfsPiece(ctx context.Context, ref cid.Cid, r UnixfsReader, dealID uint64) (sectorID uint64, err error) { size, err := r.Size() if err != nil { return 0, err @@ -166,7 +173,9 @@ func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint6 intermediate: st.intermediate, } - return st.Store.AddPiece(refst.pieceRef, uint64(size), refst, dealID) + pr, psize := padreader.New(r, uint64(size)) + + return st.Miner.SealPiece(ctx, refst.pieceRef, psize, pr, dealID) } func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { @@ -187,12 +196,12 @@ func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { return nil, err } - var refs []api.SealedRef - if err := cbor.DecodeInto(ent.Value, &refs); err != nil { + var refs api.SealedRefs + if err := cborutil.ReadCborRPC(bytes.NewReader(ent.Value), &refs); err != nil { return nil, err } - out[refCid] = refs + out[refCid] = refs.Refs } return out, nil @@ -207,12 +216,12 @@ func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: t return nil, err } - var refs []api.SealedRef - if err := cbor.DecodeInto(ent, &refs); err != nil { + var refs api.SealedRefs + if err := cborutil.ReadCborRPC(bytes.NewReader(ent), &refs); err != nil { return nil, err } - return refs, nil + return refs.Refs, nil } func (st *SectorBlocks) GetSize(k cid.Cid) (uint64, error) {